Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/bitbucket/tasks.py
| Show All 11 Lines | |||||
| GROUP_SPLIT = 10000 | GROUP_SPLIT = 10000 | ||||
| def new_lister(api_baseurl='https://api.bitbucket.org/2.0', per_page=100): | def new_lister(api_baseurl='https://api.bitbucket.org/2.0', per_page=100): | ||||
| return BitBucketLister(api_baseurl=api_baseurl, per_page=per_page) | return BitBucketLister(api_baseurl=api_baseurl, per_page=per_page) | ||||
| @app.task(name=__name__ + '.IncrementalBitBucketLister') | @app.task(name=__name__ + '.IncrementalBitBucketLister') | ||||
| def incremental_bitbucket_lister(**lister_args): | def list_bitbucket_incremental(**lister_args): | ||||
| '''Incremental update of the BitBucket forge''' | |||||
| lister = new_lister(**lister_args) | lister = new_lister(**lister_args) | ||||
| lister.run(min_bound=lister.db_last_index(), max_bound=None) | lister.run(min_bound=lister.db_last_index(), max_bound=None) | ||||
| @app.task(name=__name__ + '.RangeBitBucketLister') | @app.task(name=__name__ + '.RangeBitBucketLister') | ||||
| def range_bitbucket_lister(start, end, **lister_args): | def _range_bitbucket_lister(start, end, **lister_args): | ||||
| lister = new_lister(**lister_args) | lister = new_lister(**lister_args) | ||||
| lister.run(min_bound=start, max_bound=end) | lister.run(min_bound=start, max_bound=end) | ||||
| @app.task(name=__name__ + '.FullBitBucketRelister', bind=True) | @app.task(name=__name__ + '.FullBitBucketRelister', bind=True) | ||||
| def full_bitbucket_relister(self, split=None, **lister_args): | def list_bitbucket_full(self, split=None, **lister_args): | ||||
| """Relist from the beginning of what's already been listed. | """Full update of the BitBucket forge | ||||
| It's not to be called for an initial listing. | It's not to be called for an initial listing. | ||||
| """ | """ | ||||
| lister = new_lister(**lister_args) | lister = new_lister(**lister_args) | ||||
| ranges = lister.db_partition_indices(split or GROUP_SPLIT) | ranges = lister.db_partition_indices(split or GROUP_SPLIT) | ||||
| if not ranges: | if not ranges: | ||||
| self.log.info('Nothing to list') | self.log.info('Nothing to list') | ||||
| return | return | ||||
| random.shuffle(ranges) | random.shuffle(ranges) | ||||
| promise = group(range_bitbucket_lister.s(minv, maxv, **lister_args) | promise = group(_range_bitbucket_lister.s(minv, maxv, **lister_args) | ||||
| for minv, maxv in ranges)() | for minv, maxv in ranges)() | ||||
| self.log.debug('%s OK (spawned %s subtasks)', (self.name, len(ranges))) | self.log.debug('%s OK (spawned %s subtasks)', (self.name, len(ranges))) | ||||
| try: | try: | ||||
| promise.save() # so that we can restore the GroupResult in tests | promise.save() # so that we can restore the GroupResult in tests | ||||
| except (NotImplementedError, AttributeError): | except (NotImplementedError, AttributeError): | ||||
| self.log.info('Unable to call save_group with current result backend.') | self.log.info('Unable to call save_group with current result backend.') | ||||
| return promise.id | return promise.id | ||||
| @app.task(name=__name__ + '.ping') | @app.task(name=__name__ + '.ping') | ||||
| def ping(): | def _ping(): | ||||
| return 'OK' | return 'OK' | ||||