Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/bitbucket/tasks.py
# Copyright (C) 2017-2019 the Software Heritage developers | # Copyright (C) 2017-2019 the Software Heritage developers | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import random | import random | ||||
from celery import group | from celery import group | ||||
from swh.scheduler.celery_backend.config import app | from swh.scheduler.celery_backend.config import app | ||||
from .lister import BitBucketLister | from .lister import BitBucketLister | ||||
GROUP_SPLIT = 10000 | GROUP_SPLIT = 10000 | ||||
def new_lister(api_baseurl='https://api.bitbucket.org/2.0', per_page=100): | |||||
return BitBucketLister(api_baseurl=api_baseurl, per_page=per_page) | |||||
@app.task(name=__name__ + '.IncrementalBitBucketLister') | @app.task(name=__name__ + '.IncrementalBitBucketLister') | ||||
def list_bitbucket_incremental(**lister_args): | def list_bitbucket_incremental(**lister_args): | ||||
'''Incremental update of the BitBucket forge''' | '''Incremental update of the BitBucket forge''' | ||||
lister = new_lister(**lister_args) | lister = BitBucketLister(**lister_args) | ||||
lister.run(min_bound=lister.db_last_index(), max_bound=None) | lister.run(min_bound=lister.db_last_index(), max_bound=None) | ||||
@app.task(name=__name__ + '.RangeBitBucketLister') | @app.task(name=__name__ + '.RangeBitBucketLister') | ||||
def _range_bitbucket_lister(start, end, **lister_args): | def _range_bitbucket_lister(start, end, **lister_args): | ||||
lister = new_lister(**lister_args) | lister = BitBucketLister(**lister_args) | ||||
lister.run(min_bound=start, max_bound=end) | lister.run(min_bound=start, max_bound=end) | ||||
@app.task(name=__name__ + '.FullBitBucketRelister', bind=True) | @app.task(name=__name__ + '.FullBitBucketRelister', bind=True) | ||||
def list_bitbucket_full(self, split=None, **lister_args): | def list_bitbucket_full(self, split=None, **lister_args): | ||||
"""Full update of the BitBucket forge | """Full update of the BitBucket forge | ||||
It's not to be called for an initial listing. | It's not to be called for an initial listing. | ||||
""" | """ | ||||
lister = new_lister(**lister_args) | lister = BitBucketLister(**lister_args) | ||||
ranges = lister.db_partition_indices(split or GROUP_SPLIT) | ranges = lister.db_partition_indices(split or GROUP_SPLIT) | ||||
if not ranges: | if not ranges: | ||||
self.log.info('Nothing to list') | self.log.info('Nothing to list') | ||||
return | return | ||||
random.shuffle(ranges) | random.shuffle(ranges) | ||||
promise = group(_range_bitbucket_lister.s(minv, maxv, **lister_args) | promise = group(_range_bitbucket_lister.s(minv, maxv, **lister_args) | ||||
for minv, maxv in ranges)() | for minv, maxv in ranges)() | ||||
Show All 11 Lines |