diff --git a/swh/scheduler/tests/updater/__init__.py b/swh/scheduler/tests/updater/__init__.py index e69de29..7806abc 100644 --- a/swh/scheduler/tests/updater/__init__.py +++ b/swh/scheduler/tests/updater/__init__.py @@ -0,0 +1,34 @@ +# Copyright (C) 2018 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from arrow import utcnow + + +class UpdaterTestUtil: + """Mixin intended for event generation purposes + + """ + def _make_event(self, event_type, name): + return { + 'type': event_type, + 'repo': { + 'name': name, + }, + 'created_at': utcnow(), + } + + def _make_events(self, events): + for event_type, repo_name in events: + yield self._make_event(event_type, repo_name) + + def _make_incomplete_event(self, event_type, name, missing_data_key): + event = self._make_event(event_type, name) + del event[missing_data_key] + return event + + def _make_incomplete_events(self, events): + for event_type, repo_name, missing_data_key in events: + yield self._make_incomplete_event(event_type, repo_name, + missing_data_key) diff --git a/swh/scheduler/tests/updater/test_consumer.py b/swh/scheduler/tests/updater/test_consumer.py index f1489c6..12e15f8 100644 --- a/swh/scheduler/tests/updater/test_consumer.py +++ b/swh/scheduler/tests/updater/test_consumer.py @@ -1,214 +1,194 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest -from arrow import utcnow from hypothesis import given from hypothesis.strategies import sampled_from, from_regex, lists, tuples, text from itertools import chain from nose.tools import istest +from swh.scheduler.tests.updater import UpdaterTestUtil + from swh.scheduler.updater.events import SWHEvent, LISTENED_EVENTS from swh.scheduler.updater.consumer import UpdaterConsumer class FakeSchedulerUpdaterBackend: def __init__(self): self.events = [] def cache_put(self, events): self.events.append(events) class FakeUpdaterConsumerBase(UpdaterConsumer): def __init__(self, backend_class=FakeSchedulerUpdaterBackend): super().__init__(backend_class=backend_class) self.connection_opened = False self.connection_closed = False self.consume_called = False self.has_events_called = False def open_connection(self): self.connection_opened = True def close_connection(self): self.connection_closed = True def convert_event(self, event): pass class FakeUpdaterConsumerRaise(FakeUpdaterConsumerBase): def has_events(self): self.has_events_called = True return True def consume_events(self): self.consume_called = True raise ValueError('Broken stuff') class UpdaterConsumerRaisingTest(unittest.TestCase): def setUp(self): self.updater = FakeUpdaterConsumerRaise() @istest def running_raise(self): """Raising during run should finish fine. """ # given self.assertEqual(self.updater.count, 0) self.assertEqual(self.updater.seen_events, set()) self.assertEqual(self.updater.events, []) # when with self.assertRaisesRegex(ValueError, 'Broken stuff'): self.updater.run() # then self.assertEqual(self.updater.count, 0) self.assertEqual(self.updater.seen_events, set()) self.assertEqual(self.updater.events, []) self.assertTrue(self.updater.connection_opened) self.assertTrue(self.updater.has_events_called) self.assertTrue(self.updater.connection_closed) self.assertTrue(self.updater.consume_called) class FakeUpdaterConsumerNoEvent(FakeUpdaterConsumerBase): def has_events(self): self.has_events_called = True return False def consume_events(self): self.consume_called = True class UpdaterConsumerNoEventTest(unittest.TestCase): def setUp(self): self.updater = FakeUpdaterConsumerNoEvent() @istest def running_does_not_consume(self): """Run with no events should do just fine""" # given self.assertEqual(self.updater.count, 0) self.assertEqual(self.updater.seen_events, set()) self.assertEqual(self.updater.events, []) # when self.updater.run() # then self.assertEqual(self.updater.count, 0) self.assertEqual(self.updater.seen_events, set()) self.assertEqual(self.updater.events, []) self.assertTrue(self.updater.connection_opened) self.assertTrue(self.updater.has_events_called) self.assertTrue(self.updater.connection_closed) self.assertFalse(self.updater.consume_called) -EVENT_KEYS = ['type', 'name', 'created_at'] +EVENT_KEYS = ['type', 'repo', 'created_at'] class FakeUpdaterConsumer(FakeUpdaterConsumerBase): def __init__(self, messages): super().__init__() self.messages = messages self.debug = False def has_events(self): self.has_events_called = True return len(self.messages) > 0 def consume_events(self): self.consume_called = True for msg in self.messages: yield msg self.messages.pop() def convert_event(self, event, keys=EVENT_KEYS): for k in keys: v = event.get(k) if v is None: return None e = { 'type': event['type'], - 'url': 'https://fake.url/%s' % event['name'], + 'url': 'https://fake.url/%s' % event['repo']['name'], 'last_seen': event['created_at'] } return SWHEvent(e) -class UpdaterConsumerWithEventTest(unittest.TestCase): - def _make_event(self, event_type, name): - return { - 'type': event_type, - 'name': name, - 'created_at': utcnow(), - } - - def _make_events(self, events): - for event_type, repo_name in events: - yield self._make_event(event_type, repo_name) - - def _make_incomplete_event(self, event_type, name, missing_data_key): - event = self._make_event(event_type, name) - del event[missing_data_key] - return event - - def _make_incomplete_events(self, events): - for event_type, repo_name, missing_data_key in events: - yield self._make_incomplete_event(event_type, repo_name, - missing_data_key) - +class UpdaterConsumerWithEventTest(UpdaterTestUtil, unittest.TestCase): @istest @given(lists(tuples(sampled_from(LISTENED_EVENTS), from_regex(r'^[a-z0-9]{5,10}/[a-z0-9]{7,12}$')), min_size=3, max_size=10), lists(tuples(text(), from_regex(r'^[a-z0-9]{5,7}/[a-z0-9]{3,10}$')), min_size=3, max_size=10), lists(tuples(sampled_from(LISTENED_EVENTS), from_regex(r'^[a-z0-9]{5,10}/[a-z0-9]{7,12}$'), sampled_from(EVENT_KEYS)), min_size=3, max_size=10)) def running(self, events, uninteresting_events, incomplete_events): """Interesting events are written to cache, dropping uninteresting ones """ # given ready_events = self._make_events(events) ready_uninteresting_events = self._make_events(uninteresting_events) ready_incomplete_events = self._make_incomplete_events( incomplete_events) updater = FakeUpdaterConsumer(list(chain( ready_events, ready_incomplete_events, ready_uninteresting_events))) self.assertEqual(updater.count, 0) self.assertEqual(updater.seen_events, set()) self.assertEqual(updater.events, []) # when updater.run() # then self.assertEqual(updater.count, 0) self.assertEqual(updater.seen_events, set()) self.assertEqual(updater.events, []) self.assertTrue(updater.connection_opened) self.assertTrue(updater.has_events_called) self.assertTrue(updater.connection_closed) self.assertTrue(updater.consume_called) self.assertEqual(updater.messages, []) # uninteresting or incomplete events are dropped self.assertTrue(len(updater.backend.events), len(events)) diff --git a/swh/scheduler/tests/updater/test_ghtorrent.py b/swh/scheduler/tests/updater/test_ghtorrent.py index f7fa9d9..63adb7d 100644 --- a/swh/scheduler/tests/updater/test_ghtorrent.py +++ b/swh/scheduler/tests/updater/test_ghtorrent.py @@ -1,173 +1,160 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest -from arrow import utcnow from hypothesis import given from hypothesis.strategies import sampled_from, from_regex from nose.tools import istest from unittest.mock import patch +from swh.scheduler.tests.updater import UpdaterTestUtil + from swh.scheduler.updater.events import SWHEvent from swh.scheduler.updater.ghtorrent import ( events, GHTorrentConsumer) def event_values(): return set(events['evt']).union(set(events['ent'])) def ghtorrentize_event_name(event_name): return '%sEvent' % event_name.capitalize() EVENT_TYPES = sorted([ghtorrentize_event_name(e) for e in event_values()]) class FakeConnection: """Fake Rabbitmq connection for test purposes """ def __init__(self, conn_string): self._conn_string = conn_string self._connect = False self._release = False def connect(self): self._connect = True return True def release(self): self._connect = False self._release = True def channel(self): self.channel = True return None -class GHTorrentConsumerTest(unittest.TestCase): +class GHTorrentConsumerTest(UpdaterTestUtil, unittest.TestCase): def setUp(self): self.fake_config = { 'conn': { 'url': 'amqp://u:p@https://somewhere:9807', }, 'debug': True, 'batch_cache_write': 10, 'rabbitmq_prefetch_read': 100, } self.consumer = GHTorrentConsumer(self.fake_config, _connection_class=FakeConnection) @istest def test_init(self): # given # check init is ok self.assertEqual(self.consumer.debug, self.fake_config['debug']) self.assertEqual(self.consumer.batch, self.fake_config['batch_cache_write']) self.assertEqual(self.consumer.prefetch_read, self.fake_config['rabbitmq_prefetch_read']) self.assertEqual(self.consumer.config, self.fake_config) @istest def test_has_events(self): self.assertTrue(self.consumer.has_events()) @istest def test_connection(self): # when self.consumer.open_connection() # then self.assertEqual(self.consumer.conn._conn_string, self.fake_config['conn']['url']) self.assertTrue(self.consumer.conn._connect) self.assertFalse(self.consumer.conn._release) # when self.consumer.close_connection() # then self.assertFalse(self.consumer.conn._connect) self.assertTrue(self.consumer.conn._release) - def _make_event(self, event_type, name): - return { - 'type': event_type, - 'repo': { - 'name': name, - }, - 'created_at': utcnow(), - } - @istest @given(sampled_from(EVENT_TYPES), from_regex(r'^[a-z0-9]{5,7}/[a-z0-9]{3,10}$')) def convert_event_ok(self, event_type, name): input_event = self._make_event(event_type, name) actual_event = self.consumer.convert_event(input_event) self.assertTrue(isinstance(actual_event, SWHEvent)) event = actual_event.get() expected_event = { 'type': event_type.lower().rstrip('Event'), 'url': 'https://github.com/%s' % name, 'last_seen': input_event['created_at'], 'rate': 1, } self.assertEqual(event, expected_event) - def _make_incomplete_event(self, event_type, name, missing_data_key): - event = self._make_event(event_type, name) - del event[missing_data_key] - return event - @istest @given(sampled_from(EVENT_TYPES), from_regex(r'^[a-z0-9]{5,7}/[a-z0-9]{3,10}$'), sampled_from(['type', 'repo', 'created_at'])) def convert_event_ko(self, event_type, name, missing_data_key): input_event = self._make_incomplete_event( event_type, name, missing_data_key) actual_converted_event = self.consumer.convert_event(input_event) self.assertIsNone(actual_converted_event) @patch('swh.scheduler.updater.ghtorrent.collect_replies') @istest def consume_events(self, mock_collect_replies): # given self.consumer.queue = 'fake-queue' # hack self.consumer.open_connection() fake_events = [ self._make_event('PushEvent', 'user/some-repo'), self._make_event('PushEvent', 'user2/some-other-repo'), ] mock_collect_replies.return_value = fake_events # when actual_events = [] for e in self.consumer.consume_events(): actual_events.append(e) # then self.assertEqual(fake_events, actual_events) mock_collect_replies.assert_called_once_with( self.consumer.conn, None, 'fake-queue', no_ack=False, limit=self.fake_config['rabbitmq_prefetch_read'] )