diff --git a/swh/lister/bitbucket/tasks.py b/swh/lister/bitbucket/tasks.py --- a/swh/lister/bitbucket/tasks.py +++ b/swh/lister/bitbucket/tasks.py @@ -3,7 +3,7 @@ # See top-level LICENSE file for more information from swh.lister.core.tasks import (IndexingDiscoveryListerTask, - IndexingRangeListerTask, + RangeListerTask, IndexingRefreshListerTask, ListerTaskBase) from .lister import BitBucketLister @@ -20,7 +20,7 @@ task_queue = 'swh_lister_bitbucket_discover' -class RangeBitBucketLister(BitBucketListerTask, IndexingRangeListerTask): +class RangeBitBucketLister(BitBucketListerTask, RangeListerTask): task_queue = 'swh_lister_bitbucket_refresh' diff --git a/swh/lister/core/tasks.py b/swh/lister/core/tasks.py --- a/swh/lister/core/tasks.py +++ b/swh/lister/core/tasks.py @@ -49,25 +49,41 @@ pass -class IndexingDiscoveryListerTask(ListerTaskBase): - def run_task(self, *args, **kwargs): - lister = self.new_lister(*args, **kwargs) - return lister.run(min_index=lister.db_last_index(), max_index=None) +# Paging/Indexing lister tasks derivatives +# (cf. {github/bitbucket/gitlab}/tasks) + +class RangeListerTask(ListerTaskBase): + """Range lister task. -class IndexingRangeListerTask(ListerTaskBase): + """ def run_task(self, start, end, *args, **kwargs): lister = self.new_lister(*args, **kwargs) return lister.run(min_index=start, max_index=end) +# Indexing Lister tasks derivatives (cf. {github/bitbucket}/tasks) + + +class IndexingDiscoveryListerTask(ListerTaskBase): + """Incremental indexing lister task. + + """ + def run_task(self, *args, **kwargs): + lister = self.new_lister(*args, **kwargs) + return lister.run(min_index=lister.db_last_index(), max_index=None) + + class IndexingRefreshListerTask(ListerTaskBase): + """Full indexing lister task. + + """ GROUP_SPLIT = 10000 def run_task(self, *args, **kwargs): lister = self.new_lister(*args, **kwargs) ranges = lister.db_partition_indices(self.GROUP_SPLIT) random.shuffle(ranges) - range_task = IndexingRangeListerTask() + range_task = RangeListerTask() group(range_task.s(minv, maxv, *args, **kwargs) for minv, maxv in ranges)() diff --git a/swh/lister/github/tasks.py b/swh/lister/github/tasks.py --- a/swh/lister/github/tasks.py +++ b/swh/lister/github/tasks.py @@ -3,7 +3,7 @@ # See top-level LICENSE file for more information from swh.lister.core.tasks import (IndexingDiscoveryListerTask, - IndexingRangeListerTask, + RangeListerTask, IndexingRefreshListerTask, ListerTaskBase) from .lister import GitHubLister @@ -19,7 +19,7 @@ task_queue = 'swh_lister_github_discover' -class RangeGitHubLister(GitHubListerTask, IndexingRangeListerTask): +class RangeGitHubLister(GitHubListerTask, RangeListerTask): task_queue = 'swh_lister_github_refresh' diff --git a/swh/lister/gitlab/tasks.py b/swh/lister/gitlab/tasks.py --- a/swh/lister/gitlab/tasks.py +++ b/swh/lister/gitlab/tasks.py @@ -2,9 +2,8 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from swh.lister.core.tasks import (IndexingDiscoveryListerTask, - IndexingRangeListerTask, - IndexingRefreshListerTask, ListerTaskBase) +from swh.lister.core.tasks import ListerTaskBase, RangeListerTask + from .lister import GitLabLister @@ -16,14 +15,10 @@ lister_name=lister_name, api_baseurl=api_baseurl) -class IncrementalGitLabDotComLister(GitLabDotComListerTask, - IndexingDiscoveryListerTask): - task_queue = 'swh_lister_gitlab_discover' - +class RangeGitLabLister(GitLabDotComListerTask, RangeListerTask): + """GitLab lister working on specified range (start, end) arguments. -class RangeGitLabLister(GitLabDotComListerTask, IndexingRangeListerTask): + """ task_queue = 'swh_lister_gitlab_refresh' -class FullGitLabRelister(GitLabDotComListerTask, IndexingRefreshListerTask): - task_queue = 'swh_lister_gitlab_refresh'