diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py --- a/swh/lister/bitbucket/lister.py +++ b/swh/lister/bitbucket/lister.py @@ -15,12 +15,22 @@ logger = logging.getLogger(__name__) +DEFAULT_BITBUCKET_PAGE = 10 + + class BitBucketLister(IndexingHttpLister): PATH_TEMPLATE = '/repositories?after=%s' MODEL = BitBucketModel LISTER_NAME = 'bitbucket' instance = 'bitbucket' + def __init__(self, api_baseurl, override_config=None, per_page=100): + super().__init__( + api_baseurl=api_baseurl, override_config=override_config) + if per_page != DEFAULT_BITBUCKET_PAGE: + self.PATH_TEMPLATE = '%s&pagelen=%s' % ( + self.PATH_TEMPLATE, per_page) + def get_model_from_repo(self, repo): return { 'uid': repo['uuid'], @@ -76,11 +86,10 @@ else: ret = lower <= inner <= upper except Exception as e: - logger.error(str(e) + ': %s, %s, %s' % - (('inner=%s%s' % (type(inner), inner)), - ('lower=%s%s' % (type(lower), lower)), - ('upper=%s%s' % (type(upper), upper))) - ) + logger.error(str(e) + ': %s, %s, %s', + ('inner=%s%s' % (type(inner), inner)), + ('lower=%s%s' % (type(lower), lower)), + ('upper=%s%s' % (type(upper), upper))) raise return ret diff --git a/swh/lister/bitbucket/tasks.py b/swh/lister/bitbucket/tasks.py --- a/swh/lister/bitbucket/tasks.py +++ b/swh/lister/bitbucket/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2018 the Software Heritage developers +# Copyright (C) 2017-2019 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -12,8 +12,8 @@ GROUP_SPLIT = 10000 -def new_lister(api_baseurl='https://api.bitbucket.org/2.0'): - return BitBucketLister(api_baseurl=api_baseurl) +def new_lister(api_baseurl='https://api.bitbucket.org/2.0', per_page=100): + return BitBucketLister(api_baseurl=api_baseurl, per_page=per_page) @app.task(name=__name__ + '.IncrementalBitBucketLister') diff --git a/swh/lister/bitbucket/tests/test_tasks.py b/swh/lister/bitbucket/tests/test_tasks.py --- a/swh/lister/bitbucket/tests/test_tasks.py +++ b/swh/lister/bitbucket/tests/test_tasks.py @@ -26,7 +26,8 @@ res.wait() assert res.successful() - lister.assert_called_once_with(api_baseurl='https://api.bitbucket.org/2.0') + lister.assert_called_once_with( + api_baseurl='https://api.bitbucket.org/2.0', per_page=100) lister.db_last_index.assert_called_once_with() lister.run.assert_called_once_with(min_bound=42, max_bound=None) @@ -44,7 +45,8 @@ res.wait() assert res.successful() - lister.assert_called_once_with(api_baseurl='https://api.bitbucket.org/2.0') + lister.assert_called_once_with( + api_baseurl='https://api.bitbucket.org/2.0', per_page=100) lister.db_last_index.assert_not_called() lister.run.assert_called_once_with(min_bound=12, max_bound=42) @@ -74,7 +76,8 @@ break sleep(1) - lister.assert_called_with(api_baseurl='https://api.bitbucket.org/2.0') + lister.assert_called_with( + api_baseurl='https://api.bitbucket.org/2.0', per_page=100) # one by the FullBitbucketRelister task # + 5 for the RangeBitbucketLister subtasks