Differential D2799 Diff 9953 swh/lister/launchpad/tasks.py

Changeset View

Standalone View

swh/lister/launchpad/tasks.py

This file was added.

				# Copyright (C) 2017-2019 the Software Heritage developers
				# License: GNU General Public License version 3, or any later version
				# See top-level LICENSE file for more information

				# import random

				from celery import group, shared_task

				from .lister import LaunchpadLister

				import random

				GROUP_SPLIT = 1000


				@shared_task(name=__name__ + '.RangeLaunchpadLister')
				def _range_launchpad_lister(start, end, **lister_args):
				lister = LaunchpadLister(**lister_args)
				return lister.run(min_bound=start, max_bound=end)


				@shared_task(name=__name__ + '.FullLaunchpadRelister', bind=True)
				def list_launchpad_full(self, split=None, **lister_args):
				"""Full update of Launchpad
				"""
				lister = LaunchpadLister(**lister_args)
				nprojects = len(lister.lp_projects)
				range_size = split or GROUP_SPLIT
				ranges = [(s, s+range_size) for s in range(0, nprojects, range_size)]
				if not ranges:
				self.log.info('Nothing to list')
				return
				if ranges[-1][1] < nprojects:
				ranges += [(ranges[-1][1], nprojects)]
				random.shuffle(ranges)
				promise = group(_range_launchpad_lister.s(minv, maxv, **lister_args)
				for minv, maxv in ranges)()
				self.log.debug('%s OK (spawned %s subtasks)' % (self.name, len(ranges)))
				try:
				promise.save() # so that we can restore the GroupResult in tests
				ardumontUnsubmitted Done Inline Actions Please remove it. ardumont: Please remove it.
				except (NotImplementedError, AttributeError):
				self.log.info('Unable to call save_group with current result backend.')
				# FIXME: what to do in terms of return here?
				return promise.id


				@shared_task(name=__name__ + '.NewLaunchpadLister', bind=True)
				def list_launchpad_new(self, split=None, **lister_args):
				"""Update new entries of Launchpad
				"""
				lister = LaunchpadLister(**lister_args)
				entries = lister.db_num_entries()
				nprojects = len(lister.lp_projects)
				max_bound = nprojects - entries
				range_size = split or GROUP_SPLIT
				ranges = [(s, s+range_size) for s in range(0, max_bound, range_size)]
				if not ranges:
				self.log.info('Nothing to list')
				return
				if ranges[-1][1] < max_bound:
				ranges += [(ranges[-1][1], max_bound)]
				random.shuffle(ranges)
				promise = group(_range_launchpad_lister.s(minv, maxv, **lister_args)
				for minv, maxv in ranges)()
				self.log.debug('%s OK (spawned %s subtasks)' % (self.name, len(ranges)))
				try:
				promise.save() # so that we can restore the GroupResult in tests
				except (NotImplementedError, AttributeError):
				self.log.info('Unable to call save_group with current result backend.')
				# FIXME: what to do in terms of return here?
				return promise.id


				@shared_task(name=__name__ + '.ping')
				def _ping():
				return 'OK'