diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py --- a/swh/lister/bitbucket/lister.py +++ b/swh/lister/bitbucket/lister.py @@ -17,6 +17,7 @@ PATH_TEMPLATE = '/repositories?after=%s' MODEL = BitBucketModel LISTER_NAME = 'bitbucket' + instance = 'bitbucket' def get_model_from_repo(self, repo): return { diff --git a/swh/lister/core/lister_base.py b/swh/lister/core/lister_base.py --- a/swh/lister/core/lister_base.py +++ b/swh/lister/core/lister_base.py @@ -239,12 +239,9 @@ @property def ADDITIONAL_CONFIG(self): # noqa: N802 return { - 'credentials': - ('list[dict]', []), - 'cache_responses': - ('bool', False), - 'cache_dir': - ('str', '~/.cache/swh/lister/%s' % self.LISTER_NAME), + 'credentials': ('dict', {}), + 'cache_responses': ('bool', False), + 'cache_dir': ('str', '~/.cache/swh/lister/%s' % self.LISTER_NAME), } INITIAL_BACKOFF = 10 diff --git a/swh/lister/core/lister_transports.py b/swh/lister/core/lister_transports.py --- a/swh/lister/core/lister_transports.py +++ b/swh/lister/core/lister_transports.py @@ -62,13 +62,19 @@ """Get the full parameters passed to requests given the transport_request identifier. + This uses credentials if any are provided. The 'credentials' + configuration is expected to be a dict of multiple levels. The first + level is the lister's name, the second is the lister's instance name. + MAY BE OVERRIDDEN if something more complex than the request headers is needed. """ params = {} params['headers'] = self.request_headers() or {} - creds = self.config['credentials'] + all_creds = self.config['credentials'] + lister_creds = all_creds.get(self.LISTER_NAME, {}) + creds = lister_creds.get(self.instance, {}) auth = random.choice(creds) if creds else None if auth: params['auth'] = (auth['username'], auth['password']) diff --git a/swh/lister/debian/lister.py b/swh/lister/debian/lister.py --- a/swh/lister/debian/lister.py +++ b/swh/lister/debian/lister.py @@ -32,6 +32,7 @@ MODEL = Package PATH_TEMPLATE = None LISTER_NAME = 'debian' + instance = 'debian' def __init__(self, override_config=None): SWHListerHttpTransport.__init__(self, api_baseurl="bogus") diff --git a/swh/lister/github/lister.py b/swh/lister/github/lister.py --- a/swh/lister/github/lister.py +++ b/swh/lister/github/lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2018 the Software Heritage developers +# Copyright (C) 2017-2019 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -14,6 +14,7 @@ MODEL = GitHubModel API_URL_INDEX_RE = re.compile(r'^.*/repositories\?since=(\d+)') LISTER_NAME = 'github' + instance = 'github' # There is only 1 instance of such lister def get_model_from_repo(self, repo): return { diff --git a/swh/lister/gitlab/lister.py b/swh/lister/gitlab/lister.py --- a/swh/lister/gitlab/lister.py +++ b/swh/lister/gitlab/lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018 the Software Heritage developers +# Copyright (C) 2018-2019 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -28,19 +28,6 @@ self.PATH_TEMPLATE = '%s&per_page=%s' % ( self.PATH_TEMPLATE, per_page) - @property - def ADDITIONAL_CONFIG(self): - """Override additional config as the 'credentials' structure change - between the ancestor classes and this class. - - cf. request_params method below - - """ - default_config = super().ADDITIONAL_CONFIG - # 'credentials' is a dict of (instance, {username, password}) dict - default_config['credentials'] = ('dict', {}) - return default_config - def request_params(self, identifier): """Get the full parameters passed to requests given the transport_request identifier. diff --git a/swh/lister/npm/lister.py b/swh/lister/npm/lister.py --- a/swh/lister/npm/lister.py +++ b/swh/lister/npm/lister.py @@ -14,6 +14,7 @@ """ MODEL = NpmModel LISTER_NAME = 'npm' + instance = 'npm' def __init__(self, api_baseurl='https://replicate.npmjs.com', per_page=1000, override_config=None): diff --git a/swh/lister/pypi/lister.py b/swh/lister/pypi/lister.py --- a/swh/lister/pypi/lister.py +++ b/swh/lister/pypi/lister.py @@ -16,6 +16,7 @@ MODEL = PyPIModel LISTER_NAME = 'pypi' PAGE = 'https://pypi.org/simple/' + instance = 'pypi' # As of today only the main pypi.org is used def __init__(self, override_config=None): ListerOnePageApiTransport .__init__(self)