diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py --- a/swh/lister/bitbucket/lister.py +++ b/swh/lister/bitbucket/lister.py @@ -15,12 +15,22 @@ logger = logging.getLogger(__name__) +DEFAULT_BITBUCKET_PAGE = 10 + + class BitBucketLister(IndexingHttpLister): PATH_TEMPLATE = '/repositories?after=%s' MODEL = BitBucketModel LISTER_NAME = 'bitbucket' instance = 'bitbucket' + def __init__(self, api_baseurl, override_config=None, per_page=100): + super().__init__( + api_baseurl=api_baseurl, override_config=override_config) + if per_page != DEFAULT_BITBUCKET_PAGE: + self.PATH_TEMPLATE = '%s&pagelen=%s' % ( + self.PATH_TEMPLATE, per_page) + def get_model_from_repo(self, repo): return { 'uid': repo['uuid'], @@ -76,7 +86,7 @@ else: ret = lower <= inner <= upper except Exception as e: - logger.error(str(e) + ': %s, %s, %s' % + logger.error(str(e) + ': %s, %s, %s', (('inner=%s%s' % (type(inner), inner)), ('lower=%s%s' % (type(lower), lower)), ('upper=%s%s' % (type(upper), upper))) diff --git a/swh/lister/bitbucket/tasks.py b/swh/lister/bitbucket/tasks.py --- a/swh/lister/bitbucket/tasks.py +++ b/swh/lister/bitbucket/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2018 the Software Heritage developers +# Copyright (C) 2017-2019 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -12,8 +12,8 @@ GROUP_SPLIT = 10000 -def new_lister(api_baseurl='https://api.bitbucket.org/2.0'): - return BitBucketLister(api_baseurl=api_baseurl) +def new_lister(api_baseurl='https://api.bitbucket.org/2.0', per_page=100): + return BitBucketLister(api_baseurl=api_baseurl, per_page=per_page) @app.task(name=__name__ + '.IncrementalBitBucketLister')