diff --git a/swh/lister/core/tests/test_abstractattribute.py b/swh/lister/core/tests/test_abstractattribute.py index 0b81d33..bfadca6 100644 --- a/swh/lister/core/tests/test_abstractattribute.py +++ b/swh/lister/core/tests/test_abstractattribute.py @@ -1,68 +1,64 @@ # Copyright (C) 2017 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import abc import unittest -from nose.tools import istest - from swh.lister.core.abstractattribute import AbstractAttribute class BaseClass(abc.ABC): v1 = AbstractAttribute v2 = AbstractAttribute() v3 = AbstractAttribute('changed docstring') v4 = 'qux' class BadSubclass1(BaseClass): pass class BadSubclass2(BaseClass): v1 = 'foo' v2 = 'bar' class BadSubclass3(BaseClass): v2 = 'bar' v3 = 'baz' class GoodSubclass(BaseClass): v1 = 'foo' v2 = 'bar' v3 = 'baz' class TestAbstractAttributes(unittest.TestCase): - @istest def test_aa(self): with self.assertRaises(TypeError): BaseClass() with self.assertRaises(TypeError): BadSubclass1() with self.assertRaises(TypeError): BadSubclass2() with self.assertRaises(TypeError): BadSubclass3() self.assertIsInstance(GoodSubclass(), GoodSubclass) gsc = GoodSubclass() self.assertEqual(gsc.v1, 'foo') self.assertEqual(gsc.v2, 'bar') self.assertEqual(gsc.v3, 'baz') self.assertEqual(gsc.v4, 'qux') - @istest def test_aa_docstrings(self): self.assertEqual(BaseClass.v1.__doc__, AbstractAttribute.__doc__) self.assertEqual(BaseClass.v2.__doc__, AbstractAttribute.__doc__) self.assertEqual(BaseClass.v3.__doc__, 'AbstractAttribute: changed docstring') diff --git a/swh/lister/core/tests/test_lister.py b/swh/lister/core/tests/test_lister.py index b2e28cd..5674d89 100644 --- a/swh/lister/core/tests/test_lister.py +++ b/swh/lister/core/tests/test_lister.py @@ -1,238 +1,229 @@ # Copyright (C) 2017-2018 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import abc import time from unittest import TestCase from unittest.mock import Mock, patch import requests_mock -from testing.postgresql import Postgresql -from nose.tools import istest from sqlalchemy import create_engine +from testing.postgresql import Postgresql from swh.lister.core.abstractattribute import AbstractAttribute def noop(*args, **kwargs): pass @requests_mock.Mocker() class HttpListerTesterBase(abc.ABC): """Base testing class for subclasses of swh.lister.core.indexing_lister.SWHIndexingHttpLister. swh.lister.core.page_by_page_lister.PageByPageHttpLister See swh.lister.github.tests.test_gh_lister for an example of how to customize for a specific listing service. """ Lister = AbstractAttribute('The lister class to test') test_re = AbstractAttribute('Compiled regex matching the server url. Must' ' capture the index value.') lister_subdir = AbstractAttribute('bitbucket, github, etc.') good_api_response_file = AbstractAttribute('Example good response body') bad_api_response_file = AbstractAttribute('Example bad response body') first_index = AbstractAttribute('First index in good_api_response') entries_per_page = AbstractAttribute('Number of results in good response') LISTER_NAME = 'fake-lister' # May need to override this if the headers are used for something def response_headers(self, request): return {} # May need to override this if the server uses non-standard rate limiting # method. # Please keep the requested retry delay reasonably low. def mock_rate_quota(self, n, request, context): self.rate_limit += 1 context.status_code = 429 context.headers['Retry-After'] = '1' return '{"error":"dummy"}' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.rate_limit = 1 self.response = None self.fl = None self.helper = None if self.__class__ != HttpListerTesterBase: self.run = TestCase.run.__get__(self, self.__class__) else: self.run = noop def request_index(self, request): m = self.test_re.search(request.path_url) if m and (len(m.groups()) > 0): return m.group(1) else: return None def mock_response(self, request, context): self.fl.reset_backoff() self.rate_limit = 1 context.status_code = 200 custom_headers = self.response_headers(request) context.headers.update(custom_headers) if self.request_index(request) == str(self.first_index): with open('swh/lister/%s/tests/%s' % (self.lister_subdir, self.good_api_response_file), 'r', encoding='utf-8') as r: return r.read() else: with open('swh/lister/%s/tests/%s' % (self.lister_subdir, self.bad_api_response_file), 'r', encoding='utf-8') as r: return r.read() def mock_limit_n_response(self, n, request, context): self.fl.reset_backoff() if self.rate_limit <= n: return self.mock_rate_quota(n, request, context) else: return self.mock_response(request, context) def mock_limit_once_response(self, request, context): return self.mock_limit_n_response(1, request, context) def mock_limit_twice_response(self, request, context): return self.mock_limit_n_response(2, request, context) def get_fl(self, override_config=None): """Retrieve an instance of fake lister (fl). """ if override_config or self.fl is None: self.fl = self.Lister(api_baseurl='https://fakeurl', override_config=override_config) self.fl.INITIAL_BACKOFF = 1 self.fl.reset_backoff() return self.fl def get_api_response(self): fl = self.get_fl() if self.response is None: self.response = fl.safely_issue_request(self.first_index) return self.response - @istest def test_is_within_bounds(self, http_mocker): fl = self.get_fl() self.assertFalse(fl.is_within_bounds(1, 2, 3)) self.assertTrue(fl.is_within_bounds(2, 1, 3)) self.assertTrue(fl.is_within_bounds(1, 1, 1)) self.assertTrue(fl.is_within_bounds(1, None, None)) self.assertTrue(fl.is_within_bounds(1, None, 2)) self.assertTrue(fl.is_within_bounds(1, 0, None)) self.assertTrue(fl.is_within_bounds("b", "a", "c")) self.assertFalse(fl.is_within_bounds("a", "b", "c")) self.assertTrue(fl.is_within_bounds("a", None, "c")) self.assertTrue(fl.is_within_bounds("a", None, None)) self.assertTrue(fl.is_within_bounds("b", "a", None)) self.assertFalse(fl.is_within_bounds("a", "b", None)) self.assertTrue(fl.is_within_bounds("aa:02", "aa:01", "aa:03")) self.assertFalse(fl.is_within_bounds("aa:12", None, "aa:03")) with self.assertRaises(TypeError): fl.is_within_bounds(1.0, "b", None) with self.assertRaises(TypeError): fl.is_within_bounds("A:B", "A::B", None) - @istest def test_api_request(self, http_mocker): http_mocker.get(self.test_re, text=self.mock_limit_twice_response) with patch.object(time, 'sleep', wraps=time.sleep) as sleepmock: self.get_api_response() self.assertEqual(sleepmock.call_count, 2) - @istest def test_repos_list(self, http_mocker): http_mocker.get(self.test_re, text=self.mock_response) li = self.get_fl().transport_response_simplified( self.get_api_response() ) self.assertIsInstance(li, list) self.assertEqual(len(li), self.entries_per_page) - @istest def test_model_map(self, http_mocker): http_mocker.get(self.test_re, text=self.mock_response) fl = self.get_fl() li = fl.transport_response_simplified(self.get_api_response()) di = li[0] self.assertIsInstance(di, dict) pubs = [k for k in vars(fl.MODEL).keys() if not k.startswith('_')] for k in pubs: if k not in ['last_seen', 'task_id', 'origin_id', 'id']: self.assertIn(k, di) def disable_storage_and_scheduler(self, fl): fl.create_missing_origins_and_tasks = Mock(return_value=None) def disable_db(self, fl): fl.winnow_models = Mock(return_value=[]) fl.db_inject_repo = Mock(return_value=fl.MODEL()) fl.disable_deleted_repo_tasks = Mock(return_value=None) - @istest def test_fetch_none_nodb(self, http_mocker): http_mocker.get(self.test_re, text=self.mock_response) fl = self.get_fl() self.disable_storage_and_scheduler(fl) self.disable_db(fl) fl.run(min_bound=1, max_bound=1) # stores no results - @istest def test_fetch_one_nodb(self, http_mocker): http_mocker.get(self.test_re, text=self.mock_response) fl = self.get_fl() self.disable_storage_and_scheduler(fl) self.disable_db(fl) fl.run(min_bound=self.first_index, max_bound=self.first_index) - @istest def test_fetch_multiple_pages_nodb(self, http_mocker): http_mocker.get(self.test_re, text=self.mock_response) fl = self.get_fl() self.disable_storage_and_scheduler(fl) self.disable_db(fl) fl.run(min_bound=self.first_index) def init_db(self, db, model): engine = create_engine(db.url()) model.metadata.create_all(engine) class HttpListerTester(HttpListerTesterBase, abc.ABC): last_index = AbstractAttribute('Last index in good_api_response') @requests_mock.Mocker() - @istest def test_fetch_multiple_pages_yesdb(self, http_mocker): http_mocker.get(self.test_re, text=self.mock_response) initdb_args = Postgresql.DEFAULT_SETTINGS['initdb_args'] initdb_args = ' '.join([initdb_args, '-E UTF-8']) db = Postgresql(initdb_args=initdb_args) fl = self.get_fl(override_config={'lister_db_url': db.url()}) self.init_db(db, fl.MODEL) self.disable_storage_and_scheduler(fl) fl.run(min_bound=self.first_index) self.assertEqual(fl.db_last_index(), self.last_index) partitions = fl.db_partition_indices(5) self.assertGreater(len(partitions), 0) for k in partitions: self.assertLessEqual(len(k), 5) self.assertGreater(len(k), 0) diff --git a/swh/lister/core/tests/test_model.py b/swh/lister/core/tests/test_model.py index b2c25e1..9f07223 100644 --- a/swh/lister/core/tests/test_model.py +++ b/swh/lister/core/tests/test_model.py @@ -1,94 +1,91 @@ # Copyright (C) 2017 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest -from nose.tools import istest from sqlalchemy import Column, Integer -from swh.lister.core.models import ModelBase, IndexingModelBase +from swh.lister.core.models import IndexingModelBase, ModelBase class BadSubclass1(ModelBase): __abstract__ = True pass class BadSubclass2(ModelBase): __abstract__ = True __tablename__ = 'foo' class BadSubclass3(BadSubclass2): __abstract__ = True pass class GoodSubclass(BadSubclass2): uid = Column(Integer, primary_key=True) indexable = Column(Integer, index=True) class IndexingBadSubclass(IndexingModelBase): __abstract__ = True pass class IndexingBadSubclass2(IndexingModelBase): __abstract__ = True __tablename__ = 'foo' class IndexingBadSubclass3(IndexingBadSubclass2): __abstract__ = True pass class IndexingGoodSubclass(IndexingModelBase): uid = Column(Integer, primary_key=True) indexable = Column(Integer, index=True) __tablename__ = 'bar' class TestModel(unittest.TestCase): - @istest def test_model_instancing(self): with self.assertRaises(TypeError): ModelBase() with self.assertRaises(TypeError): BadSubclass1() with self.assertRaises(TypeError): BadSubclass2() with self.assertRaises(TypeError): BadSubclass3() self.assertIsInstance(GoodSubclass(), GoodSubclass) gsc = GoodSubclass(uid='uid') self.assertEqual(gsc.__tablename__, 'foo') self.assertEqual(gsc.uid, 'uid') - @istest def test_indexing_model_instancing(self): with self.assertRaises(TypeError): IndexingModelBase() with self.assertRaises(TypeError): IndexingBadSubclass() with self.assertRaises(TypeError): IndexingBadSubclass2() with self.assertRaises(TypeError): IndexingBadSubclass3() self.assertIsInstance(IndexingGoodSubclass(), IndexingGoodSubclass) gsc = IndexingGoodSubclass(uid='uid', indexable='indexable') self.assertEqual(gsc.__tablename__, 'bar') self.assertEqual(gsc.uid, 'uid') self.assertEqual(gsc.indexable, 'indexable') diff --git a/swh/lister/gitlab/tests/test_gitlab_lister.py b/swh/lister/gitlab/tests/test_gitlab_lister.py index 9d17330..cbe0f4d 100644 --- a/swh/lister/gitlab/tests/test_gitlab_lister.py +++ b/swh/lister/gitlab/tests/test_gitlab_lister.py @@ -1,38 +1,37 @@ # Copyright (C) 2017-2018 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import re import unittest - from datetime import datetime, timedelta -from swh.lister.gitlab.lister import GitLabLister from swh.lister.core.tests.test_lister import HttpListerTesterBase +from swh.lister.gitlab.lister import GitLabLister class GitLabListerTester(HttpListerTesterBase, unittest.TestCase): Lister = GitLabLister test_re = re.compile(r'^.*/projects.*page=(\d+).*') lister_subdir = 'gitlab' good_api_response_file = 'api_response.json' bad_api_response_file = 'api_empty_response.json' first_index = 1 entries_per_page = 10 def response_headers(self, request): headers = {'RateLimit-Remaining': '1'} if self.request_index(request) == str(self.first_index): headers.update({ 'x-next-page': '3', }) return headers def mock_rate_quota(self, n, request, context): self.rate_limit += 1 context.status_code = 403 context.headers['RateLimit-Remaining'] = '0' one_second = int((datetime.now() + timedelta(seconds=1.5)).timestamp()) context.headers['RateLimit-Reset'] = str(one_second) return '{"error":"dummy"}' diff --git a/swh/lister/tests/test_utils.py b/swh/lister/tests/test_utils.py index 978127a..5d9f476 100644 --- a/swh/lister/tests/test_utils.py +++ b/swh/lister/tests/test_utils.py @@ -1,28 +1,24 @@ # Copyright (C) 2018 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest -from nose.tools import istest - from swh.lister import utils class UtilsTest(unittest.TestCase): - @istest - def split_range(self): + def test_split_range(self): actual_ranges = list(utils.split_range(14, 5)) self.assertEqual(actual_ranges, [(0, 5), (5, 10), (10, 14)]) actual_ranges = list(utils.split_range(19, 10)) self.assertEqual(actual_ranges, [(0, 10), (10, 19)]) - @istest - def split_range_errors(self): + def test_split_range_errors(self): with self.assertRaises(TypeError): list(utils.split_range(None, 1)) with self.assertRaises(TypeError): list(utils.split_range(100, None))