Page MenuHomeSoftware Heritage
Files F9311861

D2980.id10659.diff
No OneTemporary
Actions

D2980.id10659.diff
View Options

diff --git a/swh/storage/buffer.py b/swh/storage/buffer.py
--- a/swh/storage/buffer.py
+++ b/swh/storage/buffer.py
@@ -101,9 +101,9 @@
)
def flush(self, object_types: Optional[Iterable[str]] = None) -> Dict:
+ summary: Dict[str, int] = self.storage.flush(object_types)
if object_types is None:
object_types = self.object_types
- summary = {} # type: Dict[str, Dict]
for object_type in object_types:
buffer_ = self._objects[object_type]
batches = grouper(buffer_.values(), n=self.min_batch_size[object_type])
diff --git a/swh/storage/cassandra/storage.py b/swh/storage/cassandra/storage.py
--- a/swh/storage/cassandra/storage.py
+++ b/swh/storage/cassandra/storage.py
@@ -1008,3 +1008,6 @@
"""
return None
+
+ def flush(self, object_types: Optional[Iterable[str]] = None) -> Dict:
+ return {}
diff --git a/swh/storage/filter.py b/swh/storage/filter.py
--- a/swh/storage/filter.py
+++ b/swh/storage/filter.py
@@ -148,3 +148,6 @@
self.objects_seen[object_type] = set()
return self.storage.clear_buffers(object_types)
+
+ def flush(self, object_types: Optional[Iterable[str]] = None) -> Dict:
+ return self.storage.flush(object_types)
diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py
--- a/swh/storage/in_memory.py
+++ b/swh/storage/in_memory.py
@@ -1022,3 +1022,6 @@
"""
return None
+
+ def flush(self, object_types: Optional[Iterable[str]] = None) -> Dict:
+ return {}
diff --git a/swh/storage/interface.py b/swh/storage/interface.py
--- a/swh/storage/interface.py
+++ b/swh/storage/interface.py
@@ -1311,4 +1311,10 @@
state.
"""
+
+ def flush(self, object_types: Optional[Iterable[str]] = None) -> Dict:
+ """For backend storages (pg, storage, in-memory), this is expected to be a noop
+ operation. For proxy storages (especially buffer), this is expected to trigger
+ actual writes to the backend.
+ """
...
diff --git a/swh/storage/retry.py b/swh/storage/retry.py
--- a/swh/storage/retry.py
+++ b/swh/storage/retry.py
@@ -168,14 +168,11 @@
def snapshot_add(self, snapshots: Iterable[Snapshot]) -> Dict:
return self.storage.snapshot_add(snapshots)
- @swh_retry
+ def clear_buffers(self, object_types: Optional[Iterable[str]] = None) -> None:
+ return self.storage.clear_buffers(object_types)
+
def flush(self, object_types: Optional[Iterable[str]] = None) -> Dict:
"""Specific case for buffer proxy storage failing to flush data
"""
- if hasattr(self.storage, "flush"):
- return self.storage.flush(object_types)
- return {}
-
- def clear_buffers(self, object_types: Optional[Iterable[str]] = None) -> None:
- return self.storage.clear_buffers(object_types)
+ return self.storage.flush(object_types)
diff --git a/swh/storage/storage.py b/swh/storage/storage.py
--- a/swh/storage/storage.py
+++ b/swh/storage/storage.py
@@ -189,6 +189,7 @@
@process_metrics
def content_add(self, content: Iterable[Content]) -> Dict:
ctime = now()
+
contents = [attr.evolve(c, ctime=ctime) for c in content]
objstorage_summary = self.objstorage.content_add(contents)
@@ -1221,3 +1222,6 @@
"""
return None
+
+ def flush(self, object_types: Optional[Iterable[str]] = None) -> Dict:
+ return {}
diff --git a/swh/storage/tests/test_server.py b/swh/storage/tests/test_server.py
--- a/swh/storage/tests/test_server.py
+++ b/swh/storage/tests/test_server.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2019 The Software Heritage developers
+# Copyright (C) 2019-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -7,6 +7,7 @@
import yaml
from swh.storage.api.server import load_and_check_config
+from swh.storage import get_storage
def prepare_config_file(tmpdir, content, name="config.yml"):
@@ -71,3 +72,10 @@
cfg = load_and_check_config(config_path, type="any")
assert cfg == config
+
+
+def test_no_remote_flush_operation():
+ config = {"cls": "remote", "url": "http://localhost"}
+ storage = get_storage(**config)
+ with pytest.raises(AttributeError, match="object has no attribute 'flush'"):
+ storage.flush()
diff --git a/swh/storage/tests/test_storage.py b/swh/storage/tests/test_storage.py
--- a/swh/storage/tests/test_storage.py
+++ b/swh/storage/tests/test_storage.py
@@ -3996,3 +3996,9 @@
"""
assert swh_storage.clear_buffers() is None
+
+ def test_flush(self, swh_storage):
+ """Calling clear buffers on real storage does nothing
+
+ """
+ assert swh_storage.flush() == {}
diff --git a/swh/storage/validate.py b/swh/storage/validate.py
--- a/swh/storage/validate.py
+++ b/swh/storage/validate.py
@@ -5,7 +5,7 @@
import datetime
import contextlib
-from typing import Dict, Iterable, List, Optional
+from typing import Dict, Iterable, Optional, List
from swh.model.model import (
BaseModel,
@@ -110,3 +110,6 @@
def clear_buffers(self, object_types: Optional[Iterable[str]] = None) -> None:
return self.storage.clear_buffers(object_types)
+
+ def flush(self, object_types: Optional[Iterable[str]] = None) -> Dict:
+ return self.storage.flush(object_types)

File Metadata

Mime Type
text/plain
Expires
Wed, Jul 2, 10:34 AM (2 w, 5 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3220598

Event Timeline

About · Developer information · Legal