Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/gitlab/tasks.py
| # Copyright (C) 2018 the Software Heritage developers | # Copyright (C) 2018 the Software Heritage developers | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import random | import random | ||||
| from celery import group | from celery import group | ||||
| from swh.scheduler.celery_backend.config import app | from swh.scheduler.celery_backend.config import app | ||||
| from .. import utils | from .. import utils | ||||
| from .lister import GitLabLister | from .lister import GitLabLister | ||||
| NBPAGES = 10 | NBPAGES = 10 | ||||
| def new_lister(api_baseurl='https://gitlab.com/api/v4', | |||||
| instance=None, sort='asc', per_page=20): | |||||
| return GitLabLister( | |||||
| api_baseurl=api_baseurl, instance=instance, sort=sort, | |||||
| per_page=per_page) | |||||
| @app.task(name=__name__ + '.IncrementalGitLabLister') | @app.task(name=__name__ + '.IncrementalGitLabLister') | ||||
| def list_gitlab_incremental(**lister_args): | def list_gitlab_incremental(**lister_args): | ||||
| """Incremental update of a GitLab instance""" | """Incremental update of a GitLab instance""" | ||||
| lister_args['sort'] = 'desc' | lister_args['sort'] = 'desc' | ||||
| lister = new_lister(**lister_args) | lister = GitLabLister(**lister_args) | ||||
| total_pages = lister.get_pages_information()[1] | total_pages = lister.get_pages_information()[1] | ||||
| # stopping as soon as existing origins for that instance are detected | # stopping as soon as existing origins for that instance are detected | ||||
| lister.run(min_bound=1, max_bound=total_pages, check_existence=True) | lister.run(min_bound=1, max_bound=total_pages, check_existence=True) | ||||
| @app.task(name=__name__ + '.RangeGitLabLister') | @app.task(name=__name__ + '.RangeGitLabLister') | ||||
| def _range_gitlab_lister(start, end, **lister_args): | def _range_gitlab_lister(start, end, **lister_args): | ||||
| lister = new_lister(**lister_args) | lister = GitLabLister(**lister_args) | ||||
| lister.run(min_bound=start, max_bound=end) | lister.run(min_bound=start, max_bound=end) | ||||
| @app.task(name=__name__ + '.FullGitLabRelister', bind=True) | @app.task(name=__name__ + '.FullGitLabRelister', bind=True) | ||||
| def list_gitlab_full(self, **lister_args): | def list_gitlab_full(self, **lister_args): | ||||
| """Full update of a GitLab instance""" | """Full update of a GitLab instance""" | ||||
| lister = new_lister(**lister_args) | lister = GitLabLister(**lister_args) | ||||
| _, total_pages, _ = lister.get_pages_information() | _, total_pages, _ = lister.get_pages_information() | ||||
| ranges = list(utils.split_range(total_pages, NBPAGES)) | ranges = list(utils.split_range(total_pages, NBPAGES)) | ||||
| random.shuffle(ranges) | random.shuffle(ranges) | ||||
| promise = group(_range_gitlab_lister.s(minv, maxv, **lister_args) | promise = group(_range_gitlab_lister.s(minv, maxv, **lister_args) | ||||
| for minv, maxv in ranges)() | for minv, maxv in ranges)() | ||||
| self.log.debug('%s OK (spawned %s subtasks)' % (self.name, len(ranges))) | self.log.debug('%s OK (spawned %s subtasks)' % (self.name, len(ranges))) | ||||
| try: | try: | ||||
| promise.save() | promise.save() | ||||
| except (NotImplementedError, AttributeError): | except (NotImplementedError, AttributeError): | ||||
| self.log.info('Unable to call save_group with current result backend.') | self.log.info('Unable to call save_group with current result backend.') | ||||
| return promise.id | return promise.id | ||||
| @app.task(name=__name__ + '.ping') | @app.task(name=__name__ + '.ping') | ||||
| def _ping(): | def _ping(): | ||||
| return 'OK' | return 'OK' | ||||