Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/retry.py
# Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import logging | import logging | ||||
import traceback | import traceback | ||||
from datetime import datetime | from datetime import datetime | ||||
from typing import Dict, Iterable, List, Optional, Union | from typing import Dict, Iterable, List, Optional, Union | ||||
from tenacity import ( | from tenacity import ( | ||||
retry, stop_after_attempt, wait_random_exponential, | retry, stop_after_attempt, wait_random_exponential, | ||||
) | ) | ||||
from swh.model.model import ( | |||||
Content, SkippedContent, Directory, Revision, Release, Snapshot, | |||||
Origin, | |||||
) | |||||
from swh.storage import get_storage | from swh.storage import get_storage | ||||
from swh.storage.exc import StorageArgumentException | from swh.storage.exc import StorageArgumentException | ||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
def should_retry_adding(retry_state) -> bool: | def should_retry_adding(retry_state) -> bool: | ||||
Show All 37 Lines | def __init__(self, storage): | ||||
self.storage = get_storage(**storage) | self.storage = get_storage(**storage) | ||||
def __getattr__(self, key): | def __getattr__(self, key): | ||||
if key == 'storage': | if key == 'storage': | ||||
raise AttributeError(key) | raise AttributeError(key) | ||||
return getattr(self.storage, key) | return getattr(self.storage, key) | ||||
@swh_retry | @swh_retry | ||||
def content_add(self, content: Iterable[Dict]) -> Dict: | def content_add(self, content: Iterable[Content]) -> Dict: | ||||
contents = list(content) | return self.storage.content_add(content) | ||||
return self.storage.content_add(contents) | |||||
@swh_retry | @swh_retry | ||||
def content_add_metadata(self, content: Iterable[Dict]) -> Dict: | def content_add_metadata(self, content: Iterable[Content]) -> Dict: | ||||
contents = list(content) | return self.storage.content_add_metadata(content) | ||||
return self.storage.content_add_metadata(contents) | |||||
@swh_retry | @swh_retry | ||||
def skipped_content_add(self, content: Iterable[Dict]) -> Dict: | def skipped_content_add(self, content: Iterable[SkippedContent]) -> Dict: | ||||
contents = list(content) | return self.storage.skipped_content_add(content) | ||||
return self.storage.skipped_content_add(contents) | |||||
@swh_retry | @swh_retry | ||||
def origin_add_one(self, origin: Dict) -> str: | def origin_add_one(self, origin: Origin) -> str: | ||||
return self.storage.origin_add_one(origin) | return self.storage.origin_add_one(origin) | ||||
@swh_retry | @swh_retry | ||||
def origin_visit_add(self, origin: Dict, | def origin_visit_add(self, origin: Dict, | ||||
date: Union[datetime, str], type: str) -> Dict: | date: Union[datetime, str], type: str) -> Dict: | ||||
return self.storage.origin_visit_add(origin, date, type) | return self.storage.origin_visit_add(origin, date, type) | ||||
@swh_retry | @swh_retry | ||||
Show All 20 Lines | class RetryingProxyStorage: | ||||
@swh_retry | @swh_retry | ||||
def origin_metadata_add( | def origin_metadata_add( | ||||
self, origin_url: str, ts: Union[str, datetime], | self, origin_url: str, ts: Union[str, datetime], | ||||
provider_id: int, tool_id: int, metadata: Dict) -> None: | provider_id: int, tool_id: int, metadata: Dict) -> None: | ||||
return self.storage.origin_metadata_add( | return self.storage.origin_metadata_add( | ||||
origin_url, ts, provider_id, tool_id, metadata) | origin_url, ts, provider_id, tool_id, metadata) | ||||
@swh_retry | @swh_retry | ||||
def directory_add(self, directories: Iterable[Dict]) -> Dict: | def directory_add(self, directories: Iterable[Directory]) -> Dict: | ||||
directories = list(directories) | |||||
return self.storage.directory_add(directories) | return self.storage.directory_add(directories) | ||||
@swh_retry | @swh_retry | ||||
def revision_add(self, revisions: Iterable[Dict]) -> Dict: | def revision_add(self, revisions: Iterable[Revision]) -> Dict: | ||||
revisions = list(revisions) | |||||
return self.storage.revision_add(revisions) | return self.storage.revision_add(revisions) | ||||
@swh_retry | @swh_retry | ||||
def release_add(self, releases: Iterable[Dict]) -> Dict: | def release_add(self, releases: Iterable[Release]) -> Dict: | ||||
releases = list(releases) | |||||
return self.storage.release_add(releases) | return self.storage.release_add(releases) | ||||
@swh_retry | @swh_retry | ||||
def snapshot_add(self, snapshot: Iterable[Dict]) -> Dict: | def snapshot_add(self, snapshots: Iterable[Snapshot]) -> Dict: | ||||
snapshots = list(snapshot) | |||||
return self.storage.snapshot_add(snapshots) | return self.storage.snapshot_add(snapshots) | ||||
@swh_retry | @swh_retry | ||||
def flush(self, object_types: Optional[Iterable[str]] = None) -> Dict: | def flush(self, object_types: Optional[Iterable[str]] = None) -> Dict: | ||||
"""Specific case for buffer proxy storage failing to flush data | """Specific case for buffer proxy storage failing to flush data | ||||
""" | """ | ||||
if hasattr(self.storage, 'flush'): | if hasattr(self.storage, 'flush'): | ||||
return self.storage.flush(object_types) | return self.storage.flush(object_types) | ||||
return {} | return {} |