diff --git a/requirements-swh-journal.txt b/requirements-swh-journal.txt new file mode 100644 --- /dev/null +++ b/requirements-swh-journal.txt @@ -0,0 +1 @@ +swh.journal >= 0.0.17 diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -56,8 +56,10 @@ setup_requires=['vcversioner'], install_requires=parse_requirements() + parse_requirements('swh'), extras_require={ - 'testing': parse_requirements('test'), + 'testing': (parse_requirements('test') + + parse_requirements('swh-journal')), 'schemata': ['SQLAlchemy'], + 'journal': parse_requirements('swh-journal'), }, vcversioner={}, include_package_data=True, diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py --- a/swh/storage/in_memory.py +++ b/swh/storage/in_memory.py @@ -22,7 +22,7 @@ from swh.objstorage import get_objstorage from swh.objstorage.exc import ObjNotFoundError -from .journal_writer import get_journal_writer +from .storage import get_journal_writer # Max block size of contents to return BULK_BLOCK_CONTENT_LEN_MAX = 10000 diff --git a/swh/storage/journal_writer.py b/swh/storage/journal_writer.py deleted file mode 100644 --- a/swh/storage/journal_writer.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (C) 2019 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import copy -from multiprocessing import Manager - - -from swh.model.model import BaseModel - - -class InMemoryJournalWriter: - def __init__(self): - # Share the list of objects across processes, for RemoteAPI tests. - self.manager = Manager() - self.objects = self.manager.list() - - def write_addition(self, object_type, object_): - if isinstance(object_, BaseModel): - object_ = object_.to_dict() - self.objects.append((object_type, copy.deepcopy(object_))) - - write_update = write_addition - - def write_additions(self, object_type, objects): - for object_ in objects: - self.write_addition(object_type, object_) - - -def get_journal_writer(cls, args={}): - if cls == 'inmemory': - JournalWriter = InMemoryJournalWriter - elif cls == 'kafka': - from swh.journal.direct_writer import DirectKafkaWriter \ - as JournalWriter - else: - raise ValueError('Unknown storage class `%s`' % cls) - - return JournalWriter(**args) diff --git a/swh/storage/storage.py b/swh/storage/storage.py --- a/swh/storage/storage.py +++ b/swh/storage/storage.py @@ -20,11 +20,14 @@ from .db import Db from .exc import StorageDBError from .algos import diff -from .journal_writer import get_journal_writer from swh.model.hashutil import ALGORITHMS, hash_to_bytes from swh.objstorage import get_objstorage from swh.objstorage.exc import ObjNotFoundError +try: + from swh.journal.writer import get_journal_writer +except ImportError: + get_journal_writer = None # Max block size of contents to return @@ -61,6 +64,10 @@ self.objstorage = get_objstorage(**objstorage) if journal_writer: + if get_journal_writer is None: + raise EnvironmentError( + 'You need the swh.journal package to use the ' + 'journal_writer feature') self.journal_writer = get_journal_writer(**journal_writer) else: self.journal_writer = None diff --git a/swh/storage/tests/test_api_client.py b/swh/storage/tests/test_api_client.py --- a/swh/storage/tests/test_api_client.py +++ b/swh/storage/tests/test_api_client.py @@ -10,9 +10,7 @@ import pytest from swh.core.api.tests.server_testing import ServerTestFixture -import swh.storage.storage as storage -from swh.storage.journal_writer import \ - get_journal_writer, InMemoryJournalWriter +from swh.journal import writer from swh.storage.in_memory import Storage as InMemoryStorage from swh.storage.api.client import RemoteStorage import swh.storage.api.server as server @@ -24,13 +22,16 @@ class RemotePgStorageFixture(StorageTestDbFixture, ServerTestFixture, unittest.TestCase): def setUp(self): + journal_writer = writer.get_journal_writer(cls='inmemory') + def mock_get_journal_writer(cls, args=None): assert cls == 'inmemory' return journal_writer - server.storage = None - storage.get_journal_writer = mock_get_journal_writer - journal_writer = InMemoryJournalWriter() + self.journal_writer = journal_writer + server.storage = None + self.get_journal_writer = writer.get_journal_writer + writer.get_journal_writer = mock_get_journal_writer # ServerTestFixture needs to have self.objroot for # setUp() method, but this field is defined in @@ -64,7 +65,7 @@ def tearDown(self): super().tearDown() shutil.rmtree(self.storage_base) - storage.get_journal_writer = get_journal_writer + writer.get_journal_writer = self.get_journal_writer def reset_storage(self): excluded = {'dbversion', 'tool'}