diff --git a/swh/lister/bitbucket/tasks.py b/swh/lister/bitbucket/tasks.py --- a/swh/lister/bitbucket/tasks.py +++ b/swh/lister/bitbucket/tasks.py @@ -17,33 +17,33 @@ @app.task(name=__name__ + '.IncrementalBitBucketLister') -def list_bitbucket_incremental(**lister_args): +def list_bitbucket_incremental(*args, **kwargs): '''Incremental update of the BitBucket forge''' - lister = new_lister(**lister_args) + lister = new_lister(*args, **kwargs) lister.run(min_bound=lister.db_last_index(), max_bound=None) @app.task(name=__name__ + '.RangeBitBucketLister') -def _range_bitbucket_lister(start, end, **lister_args): - lister = new_lister(**lister_args) +def _range_bitbucket_lister(start, end, *args, **kwargs): + lister = new_lister(*args, **kwargs) lister.run(min_bound=start, max_bound=end) @app.task(name=__name__ + '.FullBitBucketRelister', bind=True) -def list_bitbucket_full(self, split=None, **lister_args): +def list_bitbucket_full(self, split=None, *args, **kwargs): """Full update of the BitBucket forge It's not to be called for an initial listing. """ - lister = new_lister(**lister_args) + lister = new_lister(*args, **kwargs) ranges = lister.db_partition_indices(split or GROUP_SPLIT) if not ranges: self.log.info('Nothing to list') return random.shuffle(ranges) - promise = group(_range_bitbucket_lister.s(minv, maxv, **lister_args) + promise = group(_range_bitbucket_lister.s(minv, maxv, *args, **kwargs) for minv, maxv in ranges)() self.log.debug('%s OK (spawned %s subtasks)', (self.name, len(ranges))) try: diff --git a/swh/lister/cgit/tasks.py b/swh/lister/cgit/tasks.py --- a/swh/lister/cgit/tasks.py +++ b/swh/lister/cgit/tasks.py @@ -8,9 +8,9 @@ @app.task(name=__name__ + '.CGitListerTask') -def list_cgit(**lister_args): +def list_cgit(*args, **kwargs): '''Lister task for CGit instances''' - CGitLister(**lister_args).run() + CGitLister(*args, **kwargs).run() @app.task(name=__name__ + '.ping') diff --git a/swh/lister/cran/tasks.py b/swh/lister/cran/tasks.py --- a/swh/lister/cran/tasks.py +++ b/swh/lister/cran/tasks.py @@ -8,9 +8,9 @@ @app.task(name=__name__ + '.CRANListerTask') -def list_cran(**lister_args): +def list_cran(*args, **kwargs): '''Lister task for the CRAN registry''' - CRANLister(**lister_args).run() + CRANLister(*args, **kwargs).run() @app.task(name=__name__ + '.ping') diff --git a/swh/lister/debian/tasks.py b/swh/lister/debian/tasks.py --- a/swh/lister/debian/tasks.py +++ b/swh/lister/debian/tasks.py @@ -8,9 +8,9 @@ @app.task(name=__name__ + '.DebianListerTask') -def list_debian_distribution(distribution, **lister_args): +def list_debian_distribution(distribution, *args, **kwargs): '''List a Debian distribution''' - DebianLister(**lister_args).run(distribution) + DebianLister(*args, **kwargs).run(distribution) @app.task(name=__name__ + '.ping') diff --git a/swh/lister/github/tasks.py b/swh/lister/github/tasks.py --- a/swh/lister/github/tasks.py +++ b/swh/lister/github/tasks.py @@ -17,32 +17,32 @@ @app.task(name=__name__ + '.IncrementalGitHubLister') -def list_github_incremental(**lister_args): +def list_github_incremental(*args, **kwargs): 'Incremental update of GitHub' - lister = new_lister(**lister_args) + lister = new_lister(*args, **kwargs) lister.run(min_bound=lister.db_last_index(), max_bound=None) @app.task(name=__name__ + '.RangeGitHubLister') -def _range_github_lister(start, end, **lister_args): - lister = new_lister(**lister_args) +def _range_github_lister(start, end, *args, **kwargs): + lister = new_lister(*args, **kwargs) lister.run(min_bound=start, max_bound=end) @app.task(name=__name__ + '.FullGitHubRelister', bind=True) -def list_github_full(self, split=None, **lister_args): +def list_github_full(self, split=None, *args, **kwargs): """Full update of GitHub It's not to be called for an initial listing. """ - lister = new_lister(**lister_args) + lister = new_lister(*args, **kwargs) ranges = lister.db_partition_indices(split or GROUP_SPLIT) if not ranges: self.log.info('Nothing to list') return random.shuffle(ranges) - promise = group(_range_github_lister.s(minv, maxv, **lister_args) + promise = group(_range_github_lister.s(minv, maxv, *args, **kwargs) for minv, maxv in ranges)() self.log.debug('%s OK (spawned %s subtasks)' % (self.name, len(ranges))) try: diff --git a/swh/lister/gitlab/tasks.py b/swh/lister/gitlab/tasks.py --- a/swh/lister/gitlab/tasks.py +++ b/swh/lister/gitlab/tasks.py @@ -14,37 +14,34 @@ NBPAGES = 10 -def new_lister(api_baseurl='https://gitlab.com/api/v4', - instance=None, sort='asc', per_page=20): - return GitLabLister( - api_baseurl=api_baseurl, instance=instance, sort=sort, - per_page=per_page) +def new_lister(*args, **kwargs): + return GitLabLister(*args, **kwargs) @app.task(name=__name__ + '.IncrementalGitLabLister') -def list_gitlab_incremental(**lister_args): +def list_gitlab_incremental(*args, **kwargs): """Incremental update of a GitLab instance""" - lister_args['sort'] = 'desc' - lister = new_lister(**lister_args) + kwargs['sort'] = 'desc' + lister = new_lister(*args, **kwargs) total_pages = lister.get_pages_information()[1] # stopping as soon as existing origins for that instance are detected lister.run(min_bound=1, max_bound=total_pages, check_existence=True) @app.task(name=__name__ + '.RangeGitLabLister') -def _range_gitlab_lister(start, end, **lister_args): - lister = new_lister(**lister_args) +def _range_gitlab_lister(start, end, *args, **kwargs): + lister = new_lister(*args, **kwargs) lister.run(min_bound=start, max_bound=end) @app.task(name=__name__ + '.FullGitLabRelister', bind=True) -def list_gitlab_full(self, **lister_args): +def list_gitlab_full(self, *args, **kwargs): """Full update of a GitLab instance""" - lister = new_lister(**lister_args) + lister = new_lister(*args, **kwargs) _, total_pages, _ = lister.get_pages_information() ranges = list(utils.split_range(total_pages, NBPAGES)) random.shuffle(ranges) - promise = group(_range_gitlab_lister.s(minv, maxv, **lister_args) + promise = group(_range_gitlab_lister.s(minv, maxv, *args, **kwargs) for minv, maxv in ranges)() self.log.debug('%s OK (spawned %s subtasks)' % (self.name, len(ranges))) try: diff --git a/swh/lister/gnu/tasks.py b/swh/lister/gnu/tasks.py --- a/swh/lister/gnu/tasks.py +++ b/swh/lister/gnu/tasks.py @@ -8,9 +8,9 @@ @app.task(name=__name__ + '.GNUListerTask') -def list_gnu_full(**lister_args): +def list_gnu_full(*args, **kwargs): 'List lister for the GNU source code archive' - GNULister(**lister_args).run() + GNULister(*args, **kwargs).run() @app.task(name=__name__ + '.ping') diff --git a/swh/lister/npm/tasks.py b/swh/lister/npm/tasks.py --- a/swh/lister/npm/tasks.py +++ b/swh/lister/npm/tasks.py @@ -41,17 +41,17 @@ @app.task(name=__name__ + '.NpmListerTask') -def list_npm_full(**lister_args): +def list_npm_full(*args, **kwargs): 'Full lister for the npm (javascript) registry' - lister = NpmLister(**lister_args) + lister = NpmLister(*args, **kwargs) with save_registry_state(lister): lister.run() @app.task(name=__name__ + '.NpmIncrementalListerTask') -def list_npm_incremental(**lister_args): +def list_npm_incremental(*args, **kwargs): 'Incremental lister for the npm (javascript) registry' - lister = NpmIncrementalLister(**lister_args) + lister = NpmIncrementalLister(*args, **kwargs) update_seq_start = get_last_update_seq(lister) with save_registry_state(lister): lister.run(min_bound=update_seq_start) diff --git a/swh/lister/packagist/tasks.py b/swh/lister/packagist/tasks.py --- a/swh/lister/packagist/tasks.py +++ b/swh/lister/packagist/tasks.py @@ -8,9 +8,9 @@ @app.task(name=__name__ + '.PackagistListerTask') -def list_packagist(**lister_args): +def list_packagist(*args, **kwargs): 'List the packagist (php) registry' - PackagistLister(**lister_args).run() + PackagistLister(*args, **kwargs).run() @app.task(name=__name__ + '.ping') diff --git a/swh/lister/phabricator/tasks.py b/swh/lister/phabricator/tasks.py --- a/swh/lister/phabricator/tasks.py +++ b/swh/lister/phabricator/tasks.py @@ -7,9 +7,9 @@ @app.task(name=__name__ + '.FullPhabricatorLister') -def list_phabricator_full(**lister_args): +def list_phabricator_full(*args, **kwargs): 'Full update of a Phabricator instance' - PhabricatorLister(**lister_args).run() + PhabricatorLister(*args, **kwargs).run() @app.task(name=__name__ + '.ping') diff --git a/swh/lister/pypi/tasks.py b/swh/lister/pypi/tasks.py --- a/swh/lister/pypi/tasks.py +++ b/swh/lister/pypi/tasks.py @@ -8,9 +8,9 @@ @app.task(name=__name__ + '.PyPIListerTask') -def list_pypi(**lister_args): +def list_pypi(*args, **kwargs): 'Full update of the PyPI (python) registry' - PyPILister(**lister_args).run() + PyPILister(*args, **kwargs).run() @app.task(name=__name__ + '.ping')