Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/gitlab/tasks.py
| Show All 16 Lines | |||||
| def new_lister(api_baseurl='https://gitlab.com/api/v4', | def new_lister(api_baseurl='https://gitlab.com/api/v4', | ||||
| instance=None, sort='asc', per_page=20): | instance=None, sort='asc', per_page=20): | ||||
| return GitLabLister( | return GitLabLister( | ||||
| api_baseurl=api_baseurl, instance=instance, sort=sort, | api_baseurl=api_baseurl, instance=instance, sort=sort, | ||||
| per_page=per_page) | per_page=per_page) | ||||
| @app.task(name=__name__ + '.IncrementalGitLabLister') | @app.task(name=__name__ + '.IncrementalGitLabLister') | ||||
| def incremental_gitlab_lister(**lister_args): | def list_gitlab_incremental(**lister_args): | ||||
| """Incremental update of a GitLab instance""" | |||||
| lister_args['sort'] = 'desc' | lister_args['sort'] = 'desc' | ||||
| lister = new_lister(**lister_args) | lister = new_lister(**lister_args) | ||||
| total_pages = lister.get_pages_information()[1] | total_pages = lister.get_pages_information()[1] | ||||
| # stopping as soon as existing origins for that instance are detected | # stopping as soon as existing origins for that instance are detected | ||||
| lister.run(min_bound=1, max_bound=total_pages, check_existence=True) | lister.run(min_bound=1, max_bound=total_pages, check_existence=True) | ||||
| @app.task(name=__name__ + '.RangeGitLabLister') | @app.task(name=__name__ + '.RangeGitLabLister') | ||||
| def range_gitlab_lister(start, end, **lister_args): | def _range_gitlab_lister(start, end, **lister_args): | ||||
| lister = new_lister(**lister_args) | lister = new_lister(**lister_args) | ||||
| lister.run(min_bound=start, max_bound=end) | lister.run(min_bound=start, max_bound=end) | ||||
| @app.task(name=__name__ + '.FullGitLabRelister', bind=True) | @app.task(name=__name__ + '.FullGitLabRelister', bind=True) | ||||
| def full_gitlab_relister(self, **lister_args): | def list_gitlab_full(self, **lister_args): | ||||
| """Full lister | """Full update of a GitLab instance""" | ||||
| This should be renamed as such. | |||||
| """ | |||||
| lister = new_lister(**lister_args) | lister = new_lister(**lister_args) | ||||
| _, total_pages, _ = lister.get_pages_information() | _, total_pages, _ = lister.get_pages_information() | ||||
| ranges = list(utils.split_range(total_pages, NBPAGES)) | ranges = list(utils.split_range(total_pages, NBPAGES)) | ||||
| random.shuffle(ranges) | random.shuffle(ranges) | ||||
| promise = group(range_gitlab_lister.s(minv, maxv, **lister_args) | promise = group(_range_gitlab_lister.s(minv, maxv, **lister_args) | ||||
| for minv, maxv in ranges)() | for minv, maxv in ranges)() | ||||
| self.log.debug('%s OK (spawned %s subtasks)' % (self.name, len(ranges))) | self.log.debug('%s OK (spawned %s subtasks)' % (self.name, len(ranges))) | ||||
| try: | try: | ||||
| promise.save() | promise.save() | ||||
| except (NotImplementedError, AttributeError): | except (NotImplementedError, AttributeError): | ||||
| self.log.info('Unable to call save_group with current result backend.') | self.log.info('Unable to call save_group with current result backend.') | ||||
| return promise.id | return promise.id | ||||
| @app.task(name=__name__ + '.ping') | @app.task(name=__name__ + '.ping') | ||||
| def ping(): | def _ping(): | ||||
| return 'OK' | return 'OK' | ||||