diff --git a/swh/lister/bitbucket/tasks.py b/swh/lister/bitbucket/tasks.py --- a/swh/lister/bitbucket/tasks.py +++ b/swh/lister/bitbucket/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017 the Software Heritage developers +# Copyright (C) 2017-2018 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -10,7 +10,7 @@ class BitBucketListerTask(ListerTaskBase): - def new_lister(self): + def new_lister(self, *args, **kwargs): return BitBucketLister(lister_name='bitbucket.com', api_baseurl='https://api.bitbucket.org/2.0') diff --git a/swh/lister/core/tasks.py b/swh/lister/core/tasks.py --- a/swh/lister/core/tasks.py +++ b/swh/lister/core/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017 the Software Heritage developers +# Copyright (C) 2017-2018 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -39,34 +39,35 @@ task_queue = AbstractAttribute('Celery Task queue name') @abc.abstractmethod - def new_lister(self): + def new_lister(self, *args, **kwargs): """Return a new lister of the appropriate type. """ pass @abc.abstractmethod - def run_task(self): + def run_task(self, *args, **kwargs): pass class IndexingDiscoveryListerTask(ListerTaskBase): - def run_task(self): - lister = self.new_lister() + def run_task(self, *args, **kwargs): + lister = self.new_lister(*args, **kwargs) return lister.run(min_index=lister.db_last_index(), max_index=None) class IndexingRangeListerTask(ListerTaskBase): - def run_task(self, start, end): - lister = self.new_lister() + def run_task(self, start, end, *args, **kwargs): + lister = self.new_lister(*args, **kwargs) return lister.run(min_index=start, max_index=end) class IndexingRefreshListerTask(ListerTaskBase): GROUP_SPLIT = 10000 - def run_task(self): - lister = self.new_lister() + def run_task(self, *args, **kwargs): + lister = self.new_lister(*args, **kwargs) ranges = lister.db_partition_indices(self.GROUP_SPLIT) random.shuffle(ranges) range_task = IndexingRangeListerTask() - group(range_task.s(minv, maxv) for minv, maxv in ranges)() + group(range_task.s(minv, maxv, *args, **kwargs) + for minv, maxv in ranges)() diff --git a/swh/lister/debian/tasks.py b/swh/lister/debian/tasks.py --- a/swh/lister/debian/tasks.py +++ b/swh/lister/debian/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017 the Software Heritage developers +# Copyright (C) 2017-2018 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -10,9 +10,9 @@ class DebianListerTask(ListerTaskBase): task_queue = 'swh_lister_debian' - def new_lister(self): + def new_lister(self, *args, **kwargs): return DebianLister() - def run_task(self, distribution): + def run_task(self, distribution, *args, **kwargs): lister = self.new_lister() return lister.run(distribution) diff --git a/swh/lister/github/tasks.py b/swh/lister/github/tasks.py --- a/swh/lister/github/tasks.py +++ b/swh/lister/github/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017 the Software Heritage developers +# Copyright (C) 2017-2018 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -10,7 +10,7 @@ class GitHubListerTask(ListerTaskBase): - def new_lister(self): + def new_lister(self, *args, **kwargs): return GitHubLister(lister_name='github.com', api_baseurl='https://api.github.com') diff --git a/swh/lister/gitlab/tasks.py b/swh/lister/gitlab/tasks.py --- a/swh/lister/gitlab/tasks.py +++ b/swh/lister/gitlab/tasks.py @@ -10,9 +10,10 @@ class GitLabDotComListerTask(ListerTaskBase): - def new_lister(self): - return GitLabLister(lister_name='gitlab.com', - api_baseurl='https://gitlab.com/api/v4') + def new_lister(self, lister_name='gitlab.com', + api_baseurl='https://gitlab.com/api/v4'): + return GitLabLister( + lister_name=lister_name, api_baseurl=api_baseurl) class IncrementalGitLabDotComLister(GitLabDotComListerTask,