Differential D1239 Diff 4034 swh/journal/tests/test_publisher_no_kafka.py

Changeset View

Standalone View

swh/journal/tests/test_publisher_no_kafka.py

This file was copied from swh/journal/tests/test_publisher.py.

# Copyright (C) 2018 The Software Heritage developers		# Copyright (C) 2018-2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

		import pytest
import unittest		import unittest

from swh.model.hashutil import hash_to_bytes		from .conftest import (
from swh.journal.publisher import JournalPublisher		JournalPublisherTest, TEST_CONFIG,
from swh.storage.in_memory import Storage		CONTENTS, REVISIONS, RELEASES, ORIGINS
		)
CONTENTS = [		from swh.journal.publisher import MANDATORY_KEYS
{
'length': 3,
'sha1': hash_to_bytes(
'34973274ccef6ab4dfaaf86599792fa9c3fe4689'),
'sha1_git': b'foo',
'blake2s256': b'bar',
'sha256': b'baz',
'status': 'visible',
},
]

COMMITTER = [
{
'id': 1,
'fullname': 'foo',
},
{
'id': 2,
'fullname': 'bar',
}
]

REVISIONS = [
{
'id': hash_to_bytes('7026b7c1a2af56521e951c01ed20f255fa054238'),
'message': b'hello',
'date': {
'timestamp': {
'seconds': 1234567891,
'microseconds': 0,
},
'offset': 120,
'negative_utc': None,
},
'committer': COMMITTER[0],
'author': COMMITTER[0],
'committer_date': None,
},
{
'id': hash_to_bytes('368a48fe15b7db2383775f97c6b247011b3f14f4'),
'message': b'hello again',
'date': {
'timestamp': {
'seconds': 1234567892,
'microseconds': 0,
},
'offset': 120,
'negative_utc': None,
},
'committer': COMMITTER[1],
'author': COMMITTER[1],
'committer_date': None,
},
]

RELEASES = [
{
'id': hash_to_bytes('d81cc0710eb6cf9efd5b920a8453e1e07157b6cd'),
'name': b'v0.0.1',
'date': {
'timestamp': {
'seconds': 1234567890,
'microseconds': 0,
},
'offset': 120,
'negative_utc': None,
},
'author': COMMITTER[0],
},
]

ORIGINS = [
{
'url': 'https://somewhere.org/den/fox',
'type': 'git',
},
{
'url': 'https://overtherainbow.org/fox/den',
'type': 'svn',
}
]

ORIGIN_VISITS = [
{
'date': '2013-05-07T04:20:39.369271+00:00',
},
{
'date': '2018-11-27T17:20:39.000000+00:00',
}
]

TEST_CONFIG = {
'brokers': ['localhost'],
'temporary_prefix': 'swh.tmp_journal.new',
'final_prefix': 'swh.journal.objects',
'consumer_id': 'swh.journal.test.publisher',
'publisher_id': 'swh.journal.test.publisher',
'object_types': ['content'],
'max_messages': 3,
}


class JournalPublisherTest(JournalPublisher):
def _prepare_storage(self, config):
self.storage = Storage()
self.storage.content_add({'data': b'42', **c} for c in CONTENTS)
self.storage.revision_add(REVISIONS)
self.storage.release_add(RELEASES)
origins = self.storage.origin_add(ORIGINS)
origin_visits = []
for i, ov in enumerate(ORIGIN_VISITS):
origin_id = origins[i]['id']
ov = self.storage.origin_visit_add(origin_id, ov['date'])
origin_visits.append(ov)
self.origins = origins
self.origin_visits = origin_visits

print("publisher.origin-visits", self.origin_visits)


class JournalPublisherNoKafkaInMemoryStorage(JournalPublisherTest):		class JournalPublisherNoKafkaInMemoryStorage(JournalPublisherTest):
"""A journal publisher with:		"""A journal publisher with:
- no kafka dependency		- no kafka dependency
- in-memory storage		- in-memory storage

"""		"""
		def check_config(self, config):
		"""No need to check the configuration here as we do not use kafka

		"""
		pass

def _prepare_journal(self, config):		def _prepare_journal(self, config):
"""No journal for now		"""No journal for now

"""		"""
pass		pass


class TestPublisher(unittest.TestCase):		class TestPublisherNoKafka(unittest.TestCase):
		"""This tests only the part not using any kafka instance

		"""
def setUp(self):		def setUp(self):
self.publisher = JournalPublisherNoKafkaInMemoryStorage(TEST_CONFIG)		self.publisher = JournalPublisherNoKafkaInMemoryStorage(TEST_CONFIG)
self.contents = [{b'sha1': c['sha1']} for c in CONTENTS]		self.contents = [{b'sha1': c['sha1']} for c in CONTENTS]
self.revisions = [{b'id': c['id']} for c in REVISIONS]		self.revisions = [{b'id': c['id']} for c in REVISIONS]
self.releases = [{b'id': c['id']} for c in RELEASES]		self.releases = [{b'id': c['id']} for c in RELEASES]
# those needs id generation from the storage		# those needs id generation from the storage
# so initialization is different than other entities		# so initialization is different than other entities
self.origins = [{b'url': o['url'],		self.origins = [{b'url': o['url'],
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Lines	def test_process_objects(self):
'content': expected_contents,		'content': expected_contents,
'revision': expected_revisions,		'revision': expected_revisions,
'release': expected_releases,		'release': expected_releases,
'origin': expected_origins,		'origin': expected_origins,
'origin_visit': expected_ovs,		'origin_visit': expected_ovs,
}		}

self.assertEqual(actual_objects, expected_objects)		self.assertEqual(actual_objects, expected_objects)


		class JournalPublisherCheckTest(JournalPublisherTest):
		"""A journal publisher with:
		- no kafka dependency
		- in-memory storage

		"""
		def _prepare_journal(self, config):
		"""No journal for now

		"""
		pass


		def test_check_config_ok():
		"""Instantiate a publisher with the right config is fine

		"""
		publisher = JournalPublisherCheckTest(TEST_CONFIG)
		assert publisher is not None


		def test_check_config_ko():
		"""Instantiate a publisher with the wrong config should raise

		"""
		for k in MANDATORY_KEYS:
		conf = TEST_CONFIG.copy()
		conf.pop(k)
		with pytest.raises(ValueError) as e:
		JournalPublisherCheckTest(conf)

		error = ('Configuration error: The following keys must be'
		' provided: %s' % (','.join([k]), ))
		assert e.value.args[0] == error