Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F7123247
D363.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
9 KB
Subscribers
None
D363.diff
View Options
diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py
--- a/swh/lister/bitbucket/lister.py
+++ b/swh/lister/bitbucket/lister.py
@@ -11,6 +11,7 @@
class BitBucketLister(SWHIndexingHttpLister):
PATH_TEMPLATE = '/repositories?after=%s'
MODEL = BitBucketModel
+ LISTER_NAME = 'bitbucket.com'
def get_model_from_repo(self, repo):
return {
diff --git a/swh/lister/bitbucket/tasks.py b/swh/lister/bitbucket/tasks.py
--- a/swh/lister/bitbucket/tasks.py
+++ b/swh/lister/bitbucket/tasks.py
@@ -11,8 +11,7 @@
class BitBucketListerTask(ListerTaskBase):
def new_lister(self):
- return BitBucketLister(lister_name='bitbucket.com',
- api_baseurl='https://api.bitbucket.org/2.0')
+ return BitBucketLister(api_baseurl='https://api.bitbucket.org/2.0')
class IncrementalBitBucketLister(BitBucketListerTask,
diff --git a/swh/lister/core/indexing_lister.py b/swh/lister/core/indexing_lister.py
--- a/swh/lister/core/indexing_lister.py
+++ b/swh/lister/core/indexing_lister.py
@@ -205,8 +205,6 @@
class SWHIndexingHttpLister(SWHListerHttpTransport, SWHIndexingLister):
"""Convenience class for ensuring right lookup and init order
when combining SWHIndexingLister and SWHListerHttpTransport."""
- def __init__(self, lister_name=None, api_baseurl=None,
- override_config=None):
+ def __init__(self, api_baseurl=None, override_config=None):
SWHListerHttpTransport.__init__(self, api_baseurl=api_baseurl)
- SWHIndexingLister.__init__(self, lister_name=lister_name,
- override_config=override_config)
+ SWHIndexingLister.__init__(self, override_config=override_config)
diff --git a/swh/lister/core/lister_base.py b/swh/lister/core/lister_base.py
--- a/swh/lister/core/lister_base.py
+++ b/swh/lister/core/lister_base.py
@@ -64,6 +64,7 @@
MODEL = AbstractAttribute('Subclass type (not instance)'
' of swh.lister.core.models.ModelBase'
' customized for a specific service.')
+ LISTER_NAME = AbstractAttribute("Lister's name")
@abc.abstractmethod
def transport_request(self, identifier):
@@ -199,30 +200,27 @@
@property
def CONFIG_BASE_FILENAME(self): # noqa: N802
- return 'lister-%s' % self.lister_name
+ return 'lister-%s' % self.LISTER_NAME
@property
def ADDITIONAL_CONFIG(self): # noqa: N802
return {
'lister_db_url':
- ('str', 'postgresql:///lister-%s' % self.lister_name),
+ ('str', 'postgresql:///lister-%s' % self.LISTER_NAME),
'credentials':
('list[dict]', []),
'cache_responses':
('bool', False),
'cache_dir':
- ('str', '~/.cache/swh/lister/%s' % self.lister_name),
+ ('str', '~/.cache/swh/lister/%s' % self.LISTER_NAME),
}
INITIAL_BACKOFF = 10
MAX_RETRIES = 7
CONN_SLEEP = 10
- def __init__(self, lister_name=None, override_config=None):
+ def __init__(self, override_config=None):
self.backoff = self.INITIAL_BACKOFF
- if lister_name is None:
- raise NameError("Every lister must be assigned a lister_name.")
- self.lister_name = lister_name # 'github?', 'bitbucket?', 'foo.com?'
self.config = self.parse_config_file(
base_filename=self.CONFIG_BASE_FILENAME,
additional_configs=[self.ADDITIONAL_CONFIG]
diff --git a/swh/lister/core/paging_lister.py b/swh/lister/core/paging_lister.py
--- a/swh/lister/core/paging_lister.py
+++ b/swh/lister/core/paging_lister.py
@@ -131,8 +131,6 @@
combining SWHPagingLister and SWHListerHttpTransport.
"""
- def __init__(self, lister_name=None, api_baseurl=None,
- override_config=None):
+ def __init__(self, api_baseurl=None, override_config=None):
SWHListerHttpTransport.__init__(self, api_baseurl=api_baseurl)
- SWHPagingLister.__init__(self, lister_name=lister_name,
- override_config=override_config)
+ SWHPagingLister.__init__(self, override_config=override_config)
diff --git a/swh/lister/core/tests/test_lister.py b/swh/lister/core/tests/test_lister.py
--- a/swh/lister/core/tests/test_lister.py
+++ b/swh/lister/core/tests/test_lister.py
@@ -39,6 +39,7 @@
first_index = AbstractAttribute('First index in good_api_response')
last_index = AbstractAttribute('Last index in good_api_response')
entries_per_page = AbstractAttribute('Number of results in good response')
+ LISTER_NAME = 'fake-lister'
# May need to override this if the headers are used for something
def response_headers(self, request):
@@ -109,8 +110,7 @@
with patch(
'swh.scheduler.backend.SchedulerBackend.reconnect', noop
):
- self.fl = self.Lister(lister_name='fakelister',
- api_baseurl='https://fakeurl',
+ self.fl = self.Lister(api_baseurl='https://fakeurl',
override_config=override_config)
self.fl.INITIAL_BACKOFF = 1
diff --git a/swh/lister/debian/lister.py b/swh/lister/debian/lister.py
--- a/swh/lister/debian/lister.py
+++ b/swh/lister/debian/lister.py
@@ -31,11 +31,11 @@
class DebianLister(SWHListerHttpTransport, SWHListerBase):
MODEL = Package
PATH_TEMPLATE = None
+ LISTER_NAME = 'debian'
def __init__(self, override_config=None):
SWHListerHttpTransport.__init__(self, api_baseurl="bogus")
- SWHListerBase.__init__(self, lister_name='debian',
- override_config=override_config)
+ SWHListerBase.__init__(self, override_config=override_config)
def transport_request(self, identifier):
"""Subvert SWHListerHttpTransport.transport_request, to try several
diff --git a/swh/lister/github/lister.py b/swh/lister/github/lister.py
--- a/swh/lister/github/lister.py
+++ b/swh/lister/github/lister.py
@@ -13,6 +13,7 @@
PATH_TEMPLATE = '/repositories?since=%d'
MODEL = GitHubModel
API_URL_INDEX_RE = re.compile(r'^.*/repositories\?since=(\d+)')
+ LISTER_NAME = 'github.com'
def get_model_from_repo(self, repo):
return {
diff --git a/swh/lister/github/tasks.py b/swh/lister/github/tasks.py
--- a/swh/lister/github/tasks.py
+++ b/swh/lister/github/tasks.py
@@ -11,8 +11,7 @@
class GitHubListerTask(ListerTaskBase):
def new_lister(self):
- return GitHubLister(lister_name='github.com',
- api_baseurl='https://api.github.com')
+ return GitHubLister(api_baseurl='https://api.github.com')
class IncrementalGitHubLister(GitHubListerTask, IndexingDiscoveryListerTask):
diff --git a/swh/lister/gitlab/lister.py b/swh/lister/gitlab/lister.py
--- a/swh/lister/gitlab/lister.py
+++ b/swh/lister/gitlab/lister.py
@@ -15,14 +15,14 @@
PATH_TEMPLATE = '/projects?page=%d&order_by=id&sort=asc&simple=true'
API_URL_INDEX_RE = re.compile(r'^.*/projects.*page=(\d+).*')
MODEL = GitLabModel
+ LISTER_NAME = 'gitlab'
- @property
- def CONFIG_BASE_FILENAME(self):
- """One gitlab lister for all instances. We discriminate between the
- origin on a per instance basis in the table.
-
- """
- return 'lister-gitlab'
+ def __init__(self, api_baseurl=None, instance=None,
+ override_config=None, sort='asc'):
+ super().__init__(api_baseurl=api_baseurl,
+ override_config=override_config)
+ self.instance = instance
+ self.PATH_TEMPLATE = '%s&sort=%s' % (self.PATH_TEMPLATE, sort)
@property
def ADDITIONAL_CONFIG(self):
@@ -32,16 +32,10 @@
cf. request_params method below
"""
- return {
- 'lister_db_url':
- ('str', 'postgresql:///lister-gitlab'),
- 'credentials': # credentials is a dict
- ('dict', {}),
- 'cache_responses':
- ('bool', False),
- 'cache_dir':
- ('str', '~/.cache/swh/lister/%s' % self.lister_name),
- }
+ default_config = super().ADDITIONAL_CONFIG
+ # 'credentials' is a dict of (instance, {username, password}) dict
+ default_config['credentials'] = ('dict', {})
+ return default_config
def request_params(self, identifier):
"""Get the full parameters passed to requests given the
@@ -67,7 +61,7 @@
# Retrieve the credentials per instance
creds = self.config['credentials']
if creds:
- creds_lister = creds[self.lister_name]
+ creds_lister = creds[self.instance]
auth = random.choice(creds_lister) if creds else None
if auth:
params['auth'] = (auth['username'], auth['password'])
@@ -75,7 +69,7 @@
def get_model_from_repo(self, repo):
return {
- 'instance': self.lister_name,
+ 'instance': self.instance,
'uid': repo['id'],
'indexable': repo['id'],
'name': repo['name'],
diff --git a/swh/lister/gitlab/tasks.py b/swh/lister/gitlab/tasks.py
--- a/swh/lister/gitlab/tasks.py
+++ b/swh/lister/gitlab/tasks.py
@@ -11,10 +11,9 @@
class GitLabListerTask(ListerTaskBase):
- def new_lister(self, lister_name='gitlab.com',
- api_baseurl='https://gitlab.com/api/v4'):
- return GitLabLister(
- lister_name=lister_name, api_baseurl=api_baseurl)
+ def new_lister(self, api_baseurl='https://gitlab.com/api/v4',
+ instance='gitlab.com'):
+ return GitLabLister(api_baseurl=api_baseurl, instance=instance)
class RangeGitLabLister(GitLabListerTask, RangeListerTask):
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Dec 18 2024, 4:50 AM (13 w, 5 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3224005
Attached To
D363: swh.lister.gitlab: Make the lister instance parametric
Event Timeline
Log In to Comment