Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/core/tests/test_lister.py
- This file was added.
# Copyright (C) 2017 the Software Heritage developers | |||||
# License: GNU General Public License version 3, or any later version | |||||
# See top-level LICENSE file for more information | |||||
from unittest import TestCase | |||||
from nose.tools import istest | |||||
from unittest.mock import patch | |||||
import requests_mock | |||||
from swh.lister.core.abstractattribute import AbstractAttribute | |||||
from datetime import datetime | |||||
import abc | |||||
@requests_mock.Mocker() | |||||
class ListerTesterBase(abc.ABC): | |||||
def noop(*args, **kwargs): | |||||
pass | |||||
Lister = AbstractAttribute('Give me a lister class!') | |||||
test_re = AbstractAttribute('Compiled regex matching the server path') | |||||
lister_subdir = AbstractAttribute('bitbucket, github, etc.') | |||||
INDEX = AbstractAttribute('Example index to query from') | |||||
ENTRIES_PER_PAGE = AbstractAttribute('How many results returned?') | |||||
# may need to override this | |||||
def response_headers(self, request): | |||||
return {} | |||||
# may need to override this | |||||
def format_response(self, r): | |||||
return r.json() | |||||
# may need to override this | |||||
# please keep the requested retry delay to ~1 second | |||||
def mock_rate_quota(self, n, request, context): | |||||
self.rate_limit += 1 | |||||
context.status_code = 429 | |||||
context.headers['Retry-After'] = '1' | |||||
return '{"error":"dummy"}' | |||||
response = None | |||||
rate_limit = 1 | |||||
fl = None | |||||
def __init__(self, *args, **kwargs): | |||||
super().__init__(*args, **kwargs) | |||||
self.helper = None | |||||
if self.__class__ != ListerTesterBase: | |||||
self.run = TestCase.run.__get__(self, self.__class__) | |||||
with patch( | |||||
'swh.scheduler.backend.SchedulerBackend.reconnect', self.noop | |||||
): | |||||
self.fl = self.Lister('fakelister', 'https://fakeurl') | |||||
self.fl.INITIAL_BACKOFF = 1 | |||||
else: | |||||
self.run = self.noop | |||||
def request_index(self, request): | |||||
m = self.test_re.search(request.path_url) | |||||
if m and (len(m.groups()) > 0): | |||||
return m.group(1) | |||||
else: | |||||
return None | |||||
def mock_response(self, request, context): | |||||
self.fl.reset_backoff() | |||||
self.rate_limit = 1 | |||||
context.status_code = 200 | |||||
custom_headers = self.response_headers(request) | |||||
context.headers.update(custom_headers) | |||||
if self.request_index(request) == str(self.INDEX): | |||||
with open('swh/lister/%s/tests/api_response.txt' % | |||||
self.lister_subdir, 'r') as r: | |||||
return r.read() | |||||
else: | |||||
with open('swh/lister/%s/tests/api_empty_response.txt' % | |||||
self.lister_subdir, 'r') as r: | |||||
return r.read() | |||||
def mock_limit_n_response(self, n, request, context): | |||||
self.fl.reset_backoff() | |||||
if self.rate_limit <= n: | |||||
return self.mock_rate_quota(n, request, context) | |||||
else: | |||||
return self.mock_response(request, context) | |||||
def mock_limit_once_response(self, request, context): | |||||
return self.mock_limit_n_response(1, request, context) | |||||
def mock_limit_twice_response(self, request, context): | |||||
return self.mock_limit_n_response(2, request, context) | |||||
def get_fl(self): | |||||
self.fl.reset_backoff() | |||||
return self.fl | |||||
def get_api_response(self): | |||||
fl = self.get_fl() | |||||
if self.response is None: | |||||
self.response = fl.api_request(fl.LIST_API_TEMPLATE % self.INDEX) | |||||
return self.response | |||||
@istest | |||||
def test_within_bounds(self, http_mocker): | |||||
fl = self.get_fl() | |||||
self.assertFalse(fl.within_bounds(1, 2, 3)) | |||||
self.assertTrue(fl.within_bounds(2, 1, 3)) | |||||
self.assertTrue(fl.within_bounds(1, 1, 1)) | |||||
self.assertTrue(fl.within_bounds(1, None, None)) | |||||
@istest | |||||
def test_api_request(self, http_mocker): | |||||
http_mocker.get(self.test_re, text=self.mock_limit_twice_response) | |||||
start = datetime.now() | |||||
with patch.object(self.fl, 'rate_limit_check', | |||||
wraps=self.fl.rate_limit_check) as ratemock: | |||||
self.get_api_response() | |||||
self.assertEqual(ratemock.call_count, 3) | |||||
secs = (datetime.now()-start).total_seconds() | |||||
self.assertAlmostEqual(secs, 2, places=0) | |||||
@istest | |||||
def test_repos_list(self, http_mocker): | |||||
http_mocker.get(self.test_re, text=self.mock_response) | |||||
li = self.get_fl().list_response_repos(self.get_api_response()) | |||||
self.assertIsInstance(li, list) | |||||
self.assertEqual(len(li), self.ENTRIES_PER_PAGE) | |||||
@istest | |||||
def test_model_map(self, http_mocker): | |||||
http_mocker.get(self.test_re, text=self.mock_response) | |||||
fl = self.get_fl() | |||||
li = fl.list_response_repos(self.get_api_response()) | |||||
di = fl.get_model_from_repo(li[0]) | |||||
self.assertIsInstance(di, dict) | |||||
pubs = [k for k in vars(fl.Model).keys() if not k.startswith('_')] | |||||
for k in pubs: | |||||
if k not in ['last_seen', 'task_id', 'origin_id']: | |||||
self.assertIn(k, di) | |||||
@istest | |||||
def test_fetch_none(self, http_mocker): | |||||
http_mocker.get(self.test_re, text=self.mock_response) | |||||
fl = self.get_fl() | |||||
with patch.object(fl, 'winnow_models', return_value=[]): | |||||
fl.fetch_pages(min_index=1, max_index=1) # stores no results | |||||
@istest | |||||
def test_fetch_one(self, http_mocker): | |||||
# no db, no scheduler | |||||
http_mocker.get(self.test_re, text=self.mock_response) | |||||
fl = self.get_fl() | |||||
with patch.object(fl, 'winnow_models', return_value=[]): | |||||
with patch.object(fl, 'create_missing_origins_and_tasks', | |||||
return_value=None): | |||||
with patch.object(fl, 'inject_repo', return_value=fl.Model()): | |||||
with patch.object(fl, 'disable_deleted_repos', | |||||
return_value=None): | |||||
fl.fetch_pages(min_index=self.INDEX, | |||||
max_index=self.INDEX) | |||||
@istest | |||||
def test_fetch_multiple_pages(self, http_mocker): | |||||
# no db, no scheduler | |||||
http_mocker.get(self.test_re, text=self.mock_response) | |||||
fl = self.get_fl() | |||||
with patch.object(fl, 'winnow_models', return_value=[]): | |||||
with patch.object(fl, 'create_missing_origins_and_tasks', | |||||
return_value=None): | |||||
with patch.object(fl, 'inject_repo', return_value=fl.Model()): | |||||
with patch.object(fl, 'disable_deleted_repos', | |||||
return_value=None): | |||||
fl.fetch_pages(min_index=self.INDEX) |