Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/gitlab/tasks.py
| # Copyright (C) 2018 the Software Heritage developers | # Copyright (C) 2018 the Software Heritage developers | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import random | import random | ||||
| from celery import group | from celery import group | ||||
| from .. import utils | |||||
| from ..core.tasks import ListerTaskBase, RangeListerTask | from ..core.tasks import ListerTaskBase, RangeListerTask | ||||
| from .lister import GitLabLister | from .lister import GitLabLister | ||||
| class GitLabListerTask(ListerTaskBase): | class GitLabListerTask(ListerTaskBase): | ||||
| def new_lister(self, api_baseurl='https://gitlab.com/api/v4', | def new_lister(self, api_baseurl='https://gitlab.com/api/v4', | ||||
| instance='gitlab.com'): | instance='gitlab.com'): | ||||
| return GitLabLister(api_baseurl=api_baseurl, instance=instance) | return GitLabLister(api_baseurl=api_baseurl, instance=instance) | ||||
| class RangeGitLabLister(GitLabListerTask, RangeListerTask): | class RangeGitLabLister(GitLabListerTask, RangeListerTask): | ||||
| """Range GitLab lister (list available origins on specified range) | """Range GitLab lister (list available origins on specified range) | ||||
| """ | """ | ||||
| task_queue = 'swh_lister_gitlab_refresh' | task_queue = 'swh_lister_gitlab_refresh' | ||||
| class FullGitLabRelister(GitLabListerTask): | class FullGitLabRelister(GitLabListerTask): | ||||
| """Full GitLab lister (list all available origins from the api). | """Full GitLab lister (list all available origins from the api). | ||||
| """ | """ | ||||
| task_queue = 'swh_lister_gitlab_refresh' | task_queue = 'swh_lister_gitlab_refresh' | ||||
| # nb pages | |||||
| nb_pages = 10 | |||||
| def run_task(self, *args, **kwargs): | def run_task(self, *args, **kwargs): | ||||
| lister = self.new_lister(*args, **kwargs) | lister = self.new_lister(*args, **kwargs) | ||||
| total, _, per_page = lister.get_pages_information() | _, total_pages, _ = lister.get_pages_information() | ||||
| ranges = list(utils.split_range(total_pages, self.nb_pages)) | |||||
| ranges = [] | |||||
| prev_index = None | |||||
| for index in range(0, total, per_page): | |||||
| if index is not None and prev_index is not None: | |||||
| ranges.append((prev_index, index)) | |||||
| prev_index = index | |||||
| random.shuffle(ranges) | random.shuffle(ranges) | ||||
| range_task = RangeGitLabLister() | range_task = RangeGitLabLister() | ||||
| group(range_task.s(minv, maxv, *args, **kwargs) | group(range_task.s(minv, maxv, *args, **kwargs) | ||||
| for minv, maxv in ranges)() | for minv, maxv in ranges)() | ||||
| class IncrementalGitLabLister(ListerTaskBase): | class IncrementalGitLabLister(ListerTaskBase): | ||||
| """Incremental GitLab lister (list only new available origins). | """Incremental GitLab lister (list only new available origins). | ||||
| Show All 16 Lines | |||||