Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/gitlab/tasks.py
| Show All 12 Lines | |||||
| class GitLabListerTask(ListerTaskBase): | class GitLabListerTask(ListerTaskBase): | ||||
| def new_lister(self, lister_name='gitlab.com', | def new_lister(self, lister_name='gitlab.com', | ||||
| api_baseurl='https://gitlab.com/api/v4'): | api_baseurl='https://gitlab.com/api/v4'): | ||||
| return GitLabLister( | return GitLabLister( | ||||
| lister_name=lister_name, api_baseurl=api_baseurl) | lister_name=lister_name, api_baseurl=api_baseurl) | ||||
| class RangeGitLabLister(GitLabListerTask, RangeListerTask): | class RangeGitLabLister(GitLabListerTask, RangeListerTask): | ||||
| """GitLab lister working on specified range (start, end) arguments. | """Range GitLab lister (list available origins on specified range) | ||||
| """ | """ | ||||
| task_queue = 'swh_lister_gitlab_refresh' | task_queue = 'swh_lister_gitlab_refresh' | ||||
| class FullGitLabRelister(GitLabListerTask): | class FullGitLabRelister(GitLabListerTask): | ||||
| """Full GitLab lister (list all available origins from the api). | |||||
| """ | |||||
| task_queue = 'swh_lister_gitlab_refresh' | task_queue = 'swh_lister_gitlab_refresh' | ||||
| def run_task(self, *args, **kwargs): | def run_task(self, *args, **kwargs): | ||||
| lister = self.new_lister(*args, **kwargs) | lister = self.new_lister(*args, **kwargs) | ||||
| total, _, per_page = lister.get_pages_information() | total, _, per_page = lister.get_pages_information() | ||||
| ranges = [] | ranges = [] | ||||
| prev_index = None | prev_index = None | ||||
| for index in range(0, total, per_page): | for index in range(0, total, per_page): | ||||
| if index is not None and prev_index is not None: | if index is not None and prev_index is not None: | ||||
| ranges.append((prev_index, index)) | ranges.append((prev_index, index)) | ||||
| prev_index = index | prev_index = index | ||||
| random.shuffle(ranges) | random.shuffle(ranges) | ||||
| range_task = RangeGitLabLister() | range_task = RangeGitLabLister() | ||||
| group(range_task.s(minv, maxv, *args, **kwargs) | group(range_task.s(minv, maxv, *args, **kwargs) | ||||
| for minv, maxv in ranges)() | for minv, maxv in ranges)() | ||||
| class IncrementalGitLabLister(ListerTaskBase): | |||||
| """Incremental GitLab lister (list only new available origins). | |||||
| """ | |||||
| task_queue = 'swh_lister_gitlab_discover' | |||||
| def new_lister(self, lister_name='gitlab.com', | |||||
| api_baseurl='https://gitlab.com/api/v4'): | |||||
| # will invert the order of the lister's result | |||||
| return GitLabLister( | |||||
| lister_name=lister_name, api_baseurl=api_baseurl, | |||||
| sort='desc') | |||||
| def run_task(self, *args, **kwargs): | |||||
| lister = self.new_lister(*args, **kwargs) | |||||
| # will check for existing data and exit when found | |||||
| return lister.run(min_index=None, max_index=None, | |||||
| check_for_presence=True) | |||||