diff --git a/requirements-swh-journal.txt b/requirements-swh-journal.txt new file mode 100644 --- /dev/null +++ b/requirements-swh-journal.txt @@ -0,0 +1 @@ +swh.journal >= 0.0.17 diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -56,8 +56,10 @@ setup_requires=['vcversioner'], install_requires=parse_requirements() + parse_requirements('swh'), extras_require={ - 'testing': parse_requirements('test'), + 'testing': (parse_requirements('test') + + parse_requirements('swh-journal')), 'schemata': ['SQLAlchemy'], + 'journal': parse_requirements('swh-journal'), }, vcversioner={}, include_package_data=True, diff --git a/swh/storage/journal_writer.py b/swh/storage/journal_writer.py --- a/swh/storage/journal_writer.py +++ b/swh/storage/journal_writer.py @@ -3,38 +3,5 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import copy -from multiprocessing import Manager - - -from swh.model.model import BaseModel - - -class InMemoryJournalWriter: - def __init__(self): - # Share the list of objects across processes, for RemoteAPI tests. - self.manager = Manager() - self.objects = self.manager.list() - - def write_addition(self, object_type, object_): - if isinstance(object_, BaseModel): - object_ = object_.to_dict() - self.objects.append((object_type, copy.deepcopy(object_))) - - write_update = write_addition - - def write_additions(self, object_type, objects): - for object_ in objects: - self.write_addition(object_type, object_) - - -def get_journal_writer(cls, args={}): - if cls == 'inmemory': - JournalWriter = InMemoryJournalWriter - elif cls == 'kafka': - from swh.journal.direct_writer import DirectKafkaWriter \ - as JournalWriter - else: - raise ValueError('Unknown storage class `%s`' % cls) - - return JournalWriter(**args) +# for bw compat +from swh.journal.writer import get_journal_writer # noqa diff --git a/swh/storage/storage.py b/swh/storage/storage.py --- a/swh/storage/storage.py +++ b/swh/storage/storage.py @@ -21,11 +21,14 @@ from .db import Db from .exc import StorageDBError from .algos import diff -from .journal_writer import get_journal_writer from swh.model.hashutil import ALGORITHMS, hash_to_bytes from swh.objstorage import get_objstorage from swh.objstorage.exc import ObjNotFoundError +try: + from swh.journal.writer import get_journal_writer +except ImportError: + get_journal_writer = None # Max block size of contents to return @@ -62,6 +65,10 @@ self.objstorage = get_objstorage(**objstorage) if journal_writer: + if get_journal_writer is None: + raise EnvironmentError( + 'You need the swh.journal package to use the ' + 'journal_writer feature') self.journal_writer = get_journal_writer(**journal_writer) else: self.journal_writer = None