diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py --- a/swh/lister/bitbucket/lister.py +++ b/swh/lister/bitbucket/lister.py @@ -11,6 +11,7 @@ class BitBucketLister(SWHIndexingHttpLister): PATH_TEMPLATE = '/repositories?after=%s' MODEL = BitBucketModel + LISTER_NAME = 'bitbucket.com' def get_model_from_repo(self, repo): return { diff --git a/swh/lister/bitbucket/tasks.py b/swh/lister/bitbucket/tasks.py --- a/swh/lister/bitbucket/tasks.py +++ b/swh/lister/bitbucket/tasks.py @@ -11,8 +11,7 @@ class BitBucketListerTask(ListerTaskBase): def new_lister(self): - return BitBucketLister(lister_name='bitbucket.com', - api_baseurl='https://api.bitbucket.org/2.0') + return BitBucketLister(api_baseurl='https://api.bitbucket.org/2.0') class IncrementalBitBucketLister(BitBucketListerTask, diff --git a/swh/lister/core/indexing_lister.py b/swh/lister/core/indexing_lister.py --- a/swh/lister/core/indexing_lister.py +++ b/swh/lister/core/indexing_lister.py @@ -205,8 +205,6 @@ class SWHIndexingHttpLister(SWHListerHttpTransport, SWHIndexingLister): """Convenience class for ensuring right lookup and init order when combining SWHIndexingLister and SWHListerHttpTransport.""" - def __init__(self, lister_name=None, api_baseurl=None, - override_config=None): + def __init__(self, api_baseurl=None, override_config=None): SWHListerHttpTransport.__init__(self, api_baseurl=api_baseurl) - SWHIndexingLister.__init__(self, lister_name=lister_name, - override_config=override_config) + SWHIndexingLister.__init__(self, override_config=override_config) diff --git a/swh/lister/core/lister_base.py b/swh/lister/core/lister_base.py --- a/swh/lister/core/lister_base.py +++ b/swh/lister/core/lister_base.py @@ -64,6 +64,7 @@ MODEL = AbstractAttribute('Subclass type (not instance)' ' of swh.lister.core.models.ModelBase' ' customized for a specific service.') + LISTER_NAME = AbstractAttribute("Lister's name") @abc.abstractmethod def transport_request(self, identifier): @@ -199,30 +200,27 @@ @property def CONFIG_BASE_FILENAME(self): # noqa: N802 - return 'lister-%s' % self.lister_name + return 'lister-%s' % self.LISTER_NAME @property def ADDITIONAL_CONFIG(self): # noqa: N802 return { 'lister_db_url': - ('str', 'postgresql:///lister-%s' % self.lister_name), + ('str', 'postgresql:///lister-%s' % self.LISTER_NAME), 'credentials': ('list[dict]', []), 'cache_responses': ('bool', False), 'cache_dir': - ('str', '~/.cache/swh/lister/%s' % self.lister_name), + ('str', '~/.cache/swh/lister/%s' % self.LISTER_NAME), } INITIAL_BACKOFF = 10 MAX_RETRIES = 7 CONN_SLEEP = 10 - def __init__(self, lister_name=None, override_config=None): + def __init__(self, override_config=None): self.backoff = self.INITIAL_BACKOFF - if lister_name is None: - raise NameError("Every lister must be assigned a lister_name.") - self.lister_name = lister_name # 'github?', 'bitbucket?', 'foo.com?' self.config = self.parse_config_file( base_filename=self.CONFIG_BASE_FILENAME, additional_configs=[self.ADDITIONAL_CONFIG] diff --git a/swh/lister/core/paging_lister.py b/swh/lister/core/paging_lister.py --- a/swh/lister/core/paging_lister.py +++ b/swh/lister/core/paging_lister.py @@ -131,8 +131,6 @@ combining SWHPagingLister and SWHListerHttpTransport. """ - def __init__(self, lister_name=None, api_baseurl=None, - override_config=None): + def __init__(self, api_baseurl=None, override_config=None): SWHListerHttpTransport.__init__(self, api_baseurl=api_baseurl) - SWHPagingLister.__init__(self, lister_name=lister_name, - override_config=override_config) + SWHPagingLister.__init__(self, override_config=override_config) diff --git a/swh/lister/core/tests/test_lister.py b/swh/lister/core/tests/test_lister.py --- a/swh/lister/core/tests/test_lister.py +++ b/swh/lister/core/tests/test_lister.py @@ -39,6 +39,7 @@ first_index = AbstractAttribute('First index in good_api_response') last_index = AbstractAttribute('Last index in good_api_response') entries_per_page = AbstractAttribute('Number of results in good response') + LISTER_NAME = 'fake-lister' # May need to override this if the headers are used for something def response_headers(self, request): @@ -109,8 +110,7 @@ with patch( 'swh.scheduler.backend.SchedulerBackend.reconnect', noop ): - self.fl = self.Lister(lister_name='fakelister', - api_baseurl='https://fakeurl', + self.fl = self.Lister(api_baseurl='https://fakeurl', override_config=override_config) self.fl.INITIAL_BACKOFF = 1 diff --git a/swh/lister/debian/lister.py b/swh/lister/debian/lister.py --- a/swh/lister/debian/lister.py +++ b/swh/lister/debian/lister.py @@ -31,11 +31,11 @@ class DebianLister(SWHListerHttpTransport, SWHListerBase): MODEL = Package PATH_TEMPLATE = None + LISTER_NAME = 'debian' def __init__(self, override_config=None): SWHListerHttpTransport.__init__(self, api_baseurl="bogus") - SWHListerBase.__init__(self, lister_name='debian', - override_config=override_config) + SWHListerBase.__init__(self, override_config=override_config) def transport_request(self, identifier): """Subvert SWHListerHttpTransport.transport_request, to try several diff --git a/swh/lister/github/lister.py b/swh/lister/github/lister.py --- a/swh/lister/github/lister.py +++ b/swh/lister/github/lister.py @@ -13,6 +13,7 @@ PATH_TEMPLATE = '/repositories?since=%d' MODEL = GitHubModel API_URL_INDEX_RE = re.compile(r'^.*/repositories\?since=(\d+)') + LISTER_NAME = 'github.com' def get_model_from_repo(self, repo): return { diff --git a/swh/lister/github/tasks.py b/swh/lister/github/tasks.py --- a/swh/lister/github/tasks.py +++ b/swh/lister/github/tasks.py @@ -11,8 +11,7 @@ class GitHubListerTask(ListerTaskBase): def new_lister(self): - return GitHubLister(lister_name='github.com', - api_baseurl='https://api.github.com') + return GitHubLister(api_baseurl='https://api.github.com') class IncrementalGitHubLister(GitHubListerTask, IndexingDiscoveryListerTask): diff --git a/swh/lister/gitlab/lister.py b/swh/lister/gitlab/lister.py --- a/swh/lister/gitlab/lister.py +++ b/swh/lister/gitlab/lister.py @@ -15,14 +15,14 @@ PATH_TEMPLATE = '/projects?page=%d&order_by=id&sort=asc&simple=true' API_URL_INDEX_RE = re.compile(r'^.*/projects.*page=(\d+).*') MODEL = GitLabModel + LISTER_NAME = 'gitlab' - @property - def CONFIG_BASE_FILENAME(self): - """One gitlab lister for all instances. We discriminate between the - origin on a per instance basis in the table. - - """ - return 'lister-gitlab' + def __init__(self, api_baseurl=None, instance=None, + override_config=None, sort='asc'): + super().__init__(api_baseurl=api_baseurl, + override_config=override_config) + self.instance = instance + self.PATH_TEMPLATE = '%s&sort=%s' % (self.PATH_TEMPLATE, sort) @property def ADDITIONAL_CONFIG(self): @@ -32,16 +32,10 @@ cf. request_params method below """ - return { - 'lister_db_url': - ('str', 'postgresql:///lister-gitlab'), - 'credentials': # credentials is a dict - ('dict', {}), - 'cache_responses': - ('bool', False), - 'cache_dir': - ('str', '~/.cache/swh/lister/%s' % self.lister_name), - } + default_config = super().ADDITIONAL_CONFIG + # 'credentials' is a dict of (instance, {username, password}) dict + default_config['credentials'] = ('dict', {}) + return default_config def request_params(self, identifier): """Get the full parameters passed to requests given the @@ -67,7 +61,7 @@ # Retrieve the credentials per instance creds = self.config['credentials'] if creds: - creds_lister = creds[self.lister_name] + creds_lister = creds[self.instance] auth = random.choice(creds_lister) if creds else None if auth: params['auth'] = (auth['username'], auth['password']) @@ -75,7 +69,7 @@ def get_model_from_repo(self, repo): return { - 'instance': self.lister_name, + 'instance': self.instance, 'uid': repo['id'], 'indexable': repo['id'], 'name': repo['name'], diff --git a/swh/lister/gitlab/tasks.py b/swh/lister/gitlab/tasks.py --- a/swh/lister/gitlab/tasks.py +++ b/swh/lister/gitlab/tasks.py @@ -11,10 +11,9 @@ class GitLabListerTask(ListerTaskBase): - def new_lister(self, lister_name='gitlab.com', - api_baseurl='https://gitlab.com/api/v4'): - return GitLabLister( - lister_name=lister_name, api_baseurl=api_baseurl) + def new_lister(self, api_baseurl='https://gitlab.com/api/v4', + instance='gitlab.com'): + return GitLabLister(api_baseurl=api_baseurl, instance=instance) class RangeGitLabLister(GitLabListerTask, RangeListerTask):