Changeset View
Changeset View
Standalone View
Standalone View
swh/journal/tests/test_publisher_no_kafka.py
- This file was copied from swh/journal/tests/test_publisher.py.
# Copyright (C) 2018 The Software Heritage developers | # Copyright (C) 2018-2019 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import pytest | |||||
import unittest | import unittest | ||||
from swh.model.hashutil import hash_to_bytes | from .conftest import ( | ||||
from swh.journal.publisher import JournalPublisher | JournalPublisherTest, TEST_CONFIG, | ||||
from swh.storage.in_memory import Storage | CONTENTS, REVISIONS, RELEASES, ORIGINS | ||||
) | |||||
CONTENTS = [ | from swh.journal.publisher import MANDATORY_KEYS | ||||
{ | |||||
'length': 3, | |||||
'sha1': hash_to_bytes( | |||||
'34973274ccef6ab4dfaaf86599792fa9c3fe4689'), | |||||
'sha1_git': b'foo', | |||||
'blake2s256': b'bar', | |||||
'sha256': b'baz', | |||||
'status': 'visible', | |||||
}, | |||||
] | |||||
COMMITTER = [ | |||||
{ | |||||
'id': 1, | |||||
'fullname': 'foo', | |||||
}, | |||||
{ | |||||
'id': 2, | |||||
'fullname': 'bar', | |||||
} | |||||
] | |||||
REVISIONS = [ | |||||
{ | |||||
'id': hash_to_bytes('7026b7c1a2af56521e951c01ed20f255fa054238'), | |||||
'message': b'hello', | |||||
'date': { | |||||
'timestamp': { | |||||
'seconds': 1234567891, | |||||
'microseconds': 0, | |||||
}, | |||||
'offset': 120, | |||||
'negative_utc': None, | |||||
}, | |||||
'committer': COMMITTER[0], | |||||
'author': COMMITTER[0], | |||||
'committer_date': None, | |||||
}, | |||||
{ | |||||
'id': hash_to_bytes('368a48fe15b7db2383775f97c6b247011b3f14f4'), | |||||
'message': b'hello again', | |||||
'date': { | |||||
'timestamp': { | |||||
'seconds': 1234567892, | |||||
'microseconds': 0, | |||||
}, | |||||
'offset': 120, | |||||
'negative_utc': None, | |||||
}, | |||||
'committer': COMMITTER[1], | |||||
'author': COMMITTER[1], | |||||
'committer_date': None, | |||||
}, | |||||
] | |||||
RELEASES = [ | |||||
{ | |||||
'id': hash_to_bytes('d81cc0710eb6cf9efd5b920a8453e1e07157b6cd'), | |||||
'name': b'v0.0.1', | |||||
'date': { | |||||
'timestamp': { | |||||
'seconds': 1234567890, | |||||
'microseconds': 0, | |||||
}, | |||||
'offset': 120, | |||||
'negative_utc': None, | |||||
}, | |||||
'author': COMMITTER[0], | |||||
}, | |||||
] | |||||
ORIGINS = [ | |||||
{ | |||||
'url': 'https://somewhere.org/den/fox', | |||||
'type': 'git', | |||||
}, | |||||
{ | |||||
'url': 'https://overtherainbow.org/fox/den', | |||||
'type': 'svn', | |||||
} | |||||
] | |||||
ORIGIN_VISITS = [ | |||||
{ | |||||
'date': '2013-05-07T04:20:39.369271+00:00', | |||||
}, | |||||
{ | |||||
'date': '2018-11-27T17:20:39.000000+00:00', | |||||
} | |||||
] | |||||
TEST_CONFIG = { | |||||
'brokers': ['localhost'], | |||||
'temporary_prefix': 'swh.tmp_journal.new', | |||||
'final_prefix': 'swh.journal.objects', | |||||
'consumer_id': 'swh.journal.test.publisher', | |||||
'publisher_id': 'swh.journal.test.publisher', | |||||
'object_types': ['content'], | |||||
'max_messages': 3, | |||||
} | |||||
class JournalPublisherTest(JournalPublisher): | |||||
def _prepare_storage(self, config): | |||||
self.storage = Storage() | |||||
self.storage.content_add({'data': b'42', **c} for c in CONTENTS) | |||||
self.storage.revision_add(REVISIONS) | |||||
self.storage.release_add(RELEASES) | |||||
origins = self.storage.origin_add(ORIGINS) | |||||
origin_visits = [] | |||||
for i, ov in enumerate(ORIGIN_VISITS): | |||||
origin_id = origins[i]['id'] | |||||
ov = self.storage.origin_visit_add(origin_id, ov['date']) | |||||
origin_visits.append(ov) | |||||
self.origins = origins | |||||
self.origin_visits = origin_visits | |||||
print("publisher.origin-visits", self.origin_visits) | |||||
class JournalPublisherNoKafkaInMemoryStorage(JournalPublisherTest): | class JournalPublisherNoKafkaInMemoryStorage(JournalPublisherTest): | ||||
"""A journal publisher with: | """A journal publisher with: | ||||
- no kafka dependency | - no kafka dependency | ||||
- in-memory storage | - in-memory storage | ||||
""" | """ | ||||
def check_config(self, config): | |||||
"""No need to check the configuration here as we do not use kafka | |||||
""" | |||||
pass | |||||
def _prepare_journal(self, config): | def _prepare_journal(self, config): | ||||
"""No journal for now | """No journal for now | ||||
""" | """ | ||||
pass | pass | ||||
class TestPublisher(unittest.TestCase): | class TestPublisherNoKafka(unittest.TestCase): | ||||
"""This tests only the part not using any kafka instance | |||||
""" | |||||
def setUp(self): | def setUp(self): | ||||
self.publisher = JournalPublisherNoKafkaInMemoryStorage(TEST_CONFIG) | self.publisher = JournalPublisherNoKafkaInMemoryStorage(TEST_CONFIG) | ||||
self.contents = [{b'sha1': c['sha1']} for c in CONTENTS] | self.contents = [{b'sha1': c['sha1']} for c in CONTENTS] | ||||
self.revisions = [{b'id': c['id']} for c in REVISIONS] | self.revisions = [{b'id': c['id']} for c in REVISIONS] | ||||
self.releases = [{b'id': c['id']} for c in RELEASES] | self.releases = [{b'id': c['id']} for c in RELEASES] | ||||
# those needs id generation from the storage | # those needs id generation from the storage | ||||
# so initialization is different than other entities | # so initialization is different than other entities | ||||
self.origins = [{b'url': o['url'], | self.origins = [{b'url': o['url'], | ||||
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Lines | def test_process_objects(self): | ||||
'content': expected_contents, | 'content': expected_contents, | ||||
'revision': expected_revisions, | 'revision': expected_revisions, | ||||
'release': expected_releases, | 'release': expected_releases, | ||||
'origin': expected_origins, | 'origin': expected_origins, | ||||
'origin_visit': expected_ovs, | 'origin_visit': expected_ovs, | ||||
} | } | ||||
self.assertEqual(actual_objects, expected_objects) | self.assertEqual(actual_objects, expected_objects) | ||||
class JournalPublisherCheckTest(JournalPublisherTest): | |||||
"""A journal publisher with: | |||||
- no kafka dependency | |||||
- in-memory storage | |||||
""" | |||||
def _prepare_journal(self, config): | |||||
"""No journal for now | |||||
""" | |||||
pass | |||||
def test_check_config_ok(): | |||||
"""Instantiate a publisher with the right config is fine | |||||
""" | |||||
publisher = JournalPublisherCheckTest(TEST_CONFIG) | |||||
assert publisher is not None | |||||
def test_check_config_ko(): | |||||
"""Instantiate a publisher with the wrong config should raise | |||||
""" | |||||
for k in MANDATORY_KEYS: | |||||
conf = TEST_CONFIG.copy() | |||||
conf.pop(k) | |||||
with pytest.raises(ValueError) as e: | |||||
JournalPublisherCheckTest(conf) | |||||
error = ('Configuration error: The following keys must be' | |||||
' provided: %s' % (','.join([k]), )) | |||||
assert e.value.args[0] == error |