Changeset View
Changeset View
Standalone View
Standalone View
swh/scheduler/tests/updater/test_ghtorrent.py
# Copyright (C) 2018 The Software Heritage developers | # Copyright (C) 2018 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import unittest | import unittest | ||||
from unittest.mock import patch | from unittest.mock import patch | ||||
from hypothesis import given | from hypothesis import given | ||||
from hypothesis.strategies import sampled_from | from hypothesis.strategies import sampled_from | ||||
from swh.scheduler.updater.events import SWHEvent | from swh.scheduler.updater.events import SWHEvent | ||||
from swh.scheduler.updater.ghtorrent import (INTERESTING_EVENT_KEYS, | from swh.scheduler.updater.ghtorrent import (INTERESTING_EVENT_KEYS, | ||||
GHTorrentConsumer, events) | GHTorrentConsumer, events) | ||||
from swh.scheduler.updater.backend import SchedulerUpdaterBackend | |||||
from . import UpdaterTestUtil, from_regex | from . import UpdaterTestUtil, from_regex | ||||
def event_values(): | def event_values(): | ||||
return set(events['evt']).union(set(events['ent'])) | return set(events['evt']).union(set(events['ent'])) | ||||
Show All 32 Lines | class FakeConnection: | ||||
def channel(self): | def channel(self): | ||||
self._channel = True | self._channel = True | ||||
return FakeChannel() | return FakeChannel() | ||||
class GHTorrentConsumerTest(UpdaterTestUtil, unittest.TestCase): | class GHTorrentConsumerTest(UpdaterTestUtil, unittest.TestCase): | ||||
def setUp(self): | def setUp(self): | ||||
self.fake_config = { | config = { | ||||
'ghtorrent': { | |||||
'rabbitmq': { | |||||
'conn': { | 'conn': { | ||||
'url': 'amqp://u:p@https://somewhere:9807', | 'url': 'amqp://u:p@https://somewhere:9807', | ||||
}, | }, | ||||
'debug': True, | 'prefetch_read': 17, | ||||
'batch_cache_write': 10, | }, | ||||
'rabbitmq_prefetch_read': 100, | 'batch_cache_write': 42, | ||||
}, | |||||
'scheduler_updater': { | |||||
'cls': 'local', | |||||
'args': { | |||||
'db': 'dbname=softwareheritage-scheduler-updater-dev', | |||||
}, | |||||
}, | |||||
} | } | ||||
self.consumer = GHTorrentConsumer(self.fake_config, | GHTorrentConsumer.connection_class = FakeConnection | ||||
_connection_class=FakeConnection) | with patch.object( | ||||
SchedulerUpdaterBackend, '__init__', return_value=None): | |||||
self.consumer = GHTorrentConsumer(**config) | |||||
def test_init(self): | @patch('swh.scheduler.updater.backend.SchedulerUpdaterBackend') | ||||
def test_init(self, mock_backend): | |||||
# given | # given | ||||
# check init is ok | # check init is ok | ||||
self.assertEqual(self.consumer.debug, | self.assertEqual(self.consumer.batch, 42) | ||||
self.fake_config['debug']) | self.assertEqual(self.consumer.prefetch_read, 17) | ||||
self.assertEqual(self.consumer.batch, | |||||
self.fake_config['batch_cache_write']) | |||||
self.assertEqual(self.consumer.prefetch_read, | |||||
self.fake_config['rabbitmq_prefetch_read']) | |||||
self.assertEqual(self.consumer.config, self.fake_config) | |||||
def test_has_events(self): | def test_has_events(self): | ||||
self.assertTrue(self.consumer.has_events()) | self.assertTrue(self.consumer.has_events()) | ||||
def test_connection(self): | def test_connection(self): | ||||
# when | # when | ||||
self.consumer.open_connection() | self.consumer.open_connection() | ||||
# then | # then | ||||
self.assertEqual(self.consumer.conn._conn_string, | self.assertEqual(self.consumer.conn._conn_string, | ||||
self.fake_config['conn']['url']) | 'amqp://u:p@https://somewhere:9807') | ||||
self.assertTrue(self.consumer.conn._connect) | self.assertTrue(self.consumer.conn._connect) | ||||
self.assertFalse(self.consumer.conn._release) | self.assertFalse(self.consumer.conn._release) | ||||
# when | # when | ||||
self.consumer.close_connection() | self.consumer.close_connection() | ||||
# then | # then | ||||
self.assertFalse(self.consumer.conn._connect) | self.assertFalse(self.consumer.conn._connect) | ||||
Show All 21 Lines | class GHTorrentConsumerTest(UpdaterTestUtil, unittest.TestCase): | ||||
@given(sampled_from(EVENT_TYPES), | @given(sampled_from(EVENT_TYPES), | ||||
from_regex(r'^[a-z0-9]{5,7}/[a-z0-9]{3,10}$'), | from_regex(r'^[a-z0-9]{5,7}/[a-z0-9]{3,10}$'), | ||||
sampled_from(INTERESTING_EVENT_KEYS)) | sampled_from(INTERESTING_EVENT_KEYS)) | ||||
def test_convert_event_ko(self, event_type, name, missing_data_key): | def test_convert_event_ko(self, event_type, name, missing_data_key): | ||||
input_event = self._make_incomplete_event( | input_event = self._make_incomplete_event( | ||||
event_type, name, 'git', missing_data_key) | event_type, name, 'git', missing_data_key) | ||||
logger = self.consumer.log | |||||
del self.consumer.log # prevent gazillions of warnings | |||||
actual_converted_event = self.consumer.convert_event(input_event) | actual_converted_event = self.consumer.convert_event(input_event) | ||||
self.consumer.log = logger | |||||
self.assertIsNone(actual_converted_event) | self.assertIsNone(actual_converted_event) | ||||
@patch('swh.scheduler.updater.ghtorrent.collect_replies') | @patch('swh.scheduler.updater.ghtorrent.collect_replies') | ||||
def test_consume_events(self, mock_collect_replies): | def test_consume_events(self, mock_collect_replies): | ||||
# given | # given | ||||
self.consumer.queue = 'fake-queue' # hack | self.consumer.queue = 'fake-queue' # hack | ||||
self.consumer.open_connection() | self.consumer.open_connection() | ||||
Show All 12 Lines | def test_consume_events(self, mock_collect_replies): | ||||
# then | # then | ||||
self.assertEqual(fake_events, actual_events) | self.assertEqual(fake_events, actual_events) | ||||
mock_collect_replies.assert_called_once_with( | mock_collect_replies.assert_called_once_with( | ||||
self.consumer.conn, | self.consumer.conn, | ||||
self.consumer.channel, | self.consumer.channel, | ||||
'fake-queue', | 'fake-queue', | ||||
no_ack=False, | no_ack=False, | ||||
limit=self.fake_config['rabbitmq_prefetch_read'] | limit=17 | ||||
) | ) |