Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F7123895
D1993.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
5 KB
Subscribers
None
D1993.diff
View Options
diff --git a/requirements-swh-journal.txt b/requirements-swh-journal.txt
new file mode 100644
--- /dev/null
+++ b/requirements-swh-journal.txt
@@ -0,0 +1 @@
+swh.journal >= 0.0.17
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -56,8 +56,10 @@
setup_requires=['vcversioner'],
install_requires=parse_requirements() + parse_requirements('swh'),
extras_require={
- 'testing': parse_requirements('test'),
+ 'testing': (parse_requirements('test') +
+ parse_requirements('swh-journal')),
'schemata': ['SQLAlchemy'],
+ 'journal': parse_requirements('swh-journal'),
},
vcversioner={},
include_package_data=True,
diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py
--- a/swh/storage/in_memory.py
+++ b/swh/storage/in_memory.py
@@ -22,7 +22,7 @@
from swh.objstorage import get_objstorage
from swh.objstorage.exc import ObjNotFoundError
-from .journal_writer import get_journal_writer
+from .storage import get_journal_writer
# Max block size of contents to return
BULK_BLOCK_CONTENT_LEN_MAX = 10000
diff --git a/swh/storage/journal_writer.py b/swh/storage/journal_writer.py
deleted file mode 100644
--- a/swh/storage/journal_writer.py
+++ /dev/null
@@ -1,40 +0,0 @@
-# Copyright (C) 2019 The Software Heritage developers
-# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
-# See top-level LICENSE file for more information
-
-import copy
-from multiprocessing import Manager
-
-
-from swh.model.model import BaseModel
-
-
-class InMemoryJournalWriter:
- def __init__(self):
- # Share the list of objects across processes, for RemoteAPI tests.
- self.manager = Manager()
- self.objects = self.manager.list()
-
- def write_addition(self, object_type, object_):
- if isinstance(object_, BaseModel):
- object_ = object_.to_dict()
- self.objects.append((object_type, copy.deepcopy(object_)))
-
- write_update = write_addition
-
- def write_additions(self, object_type, objects):
- for object_ in objects:
- self.write_addition(object_type, object_)
-
-
-def get_journal_writer(cls, args={}):
- if cls == 'inmemory':
- JournalWriter = InMemoryJournalWriter
- elif cls == 'kafka':
- from swh.journal.direct_writer import DirectKafkaWriter \
- as JournalWriter
- else:
- raise ValueError('Unknown storage class `%s`' % cls)
-
- return JournalWriter(**args)
diff --git a/swh/storage/storage.py b/swh/storage/storage.py
--- a/swh/storage/storage.py
+++ b/swh/storage/storage.py
@@ -20,11 +20,14 @@
from .db import Db
from .exc import StorageDBError
from .algos import diff
-from .journal_writer import get_journal_writer
from swh.model.hashutil import ALGORITHMS, hash_to_bytes
from swh.objstorage import get_objstorage
from swh.objstorage.exc import ObjNotFoundError
+try:
+ from swh.journal.writer import get_journal_writer
+except ImportError:
+ get_journal_writer = None
# Max block size of contents to return
@@ -61,6 +64,10 @@
self.objstorage = get_objstorage(**objstorage)
if journal_writer:
+ if get_journal_writer is None:
+ raise EnvironmentError(
+ 'You need the swh.journal package to use the '
+ 'journal_writer feature')
self.journal_writer = get_journal_writer(**journal_writer)
else:
self.journal_writer = None
diff --git a/swh/storage/tests/test_api_client.py b/swh/storage/tests/test_api_client.py
--- a/swh/storage/tests/test_api_client.py
+++ b/swh/storage/tests/test_api_client.py
@@ -10,9 +10,7 @@
import pytest
from swh.core.api.tests.server_testing import ServerTestFixture
-import swh.storage.storage as storage
-from swh.storage.journal_writer import \
- get_journal_writer, InMemoryJournalWriter
+from swh.journal import writer
from swh.storage.in_memory import Storage as InMemoryStorage
from swh.storage.api.client import RemoteStorage
import swh.storage.api.server as server
@@ -24,13 +22,16 @@
class RemotePgStorageFixture(StorageTestDbFixture, ServerTestFixture,
unittest.TestCase):
def setUp(self):
+ journal_writer = writer.get_journal_writer(cls='inmemory')
+
def mock_get_journal_writer(cls, args=None):
assert cls == 'inmemory'
return journal_writer
- server.storage = None
- storage.get_journal_writer = mock_get_journal_writer
- journal_writer = InMemoryJournalWriter()
+
self.journal_writer = journal_writer
+ server.storage = None
+ self.get_journal_writer = writer.get_journal_writer
+ writer.get_journal_writer = mock_get_journal_writer
# ServerTestFixture needs to have self.objroot for
# setUp() method, but this field is defined in
@@ -64,7 +65,7 @@
def tearDown(self):
super().tearDown()
shutil.rmtree(self.storage_base)
- storage.get_journal_writer = get_journal_writer
+ writer.get_journal_writer = self.get_journal_writer
def reset_storage(self):
excluded = {'dbversion', 'tool'}
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Dec 20 2024, 5:52 AM (11 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3222507
Attached To
D1993: journal_writer: use journal writer from swh.journal
Event Timeline
Log In to Comment