diff --git a/swh/lister/bitbucket/tasks.py b/swh/lister/bitbucket/tasks.py --- a/swh/lister/bitbucket/tasks.py +++ b/swh/lister/bitbucket/tasks.py @@ -14,13 +14,13 @@ def list_bitbucket_incremental(**lister_args): '''Incremental update of the BitBucket forge''' lister = BitBucketLister(**lister_args) - lister.run(min_bound=lister.db_last_index(), max_bound=None) + return lister.run(min_bound=lister.db_last_index(), max_bound=None) @shared_task(name=__name__ + '.RangeBitBucketLister') def _range_bitbucket_lister(start, end, **lister_args): lister = BitBucketLister(**lister_args) - lister.run(min_bound=start, max_bound=end) + return lister.run(min_bound=start, max_bound=end) @shared_task(name=__name__ + '.FullBitBucketRelister', bind=True) @@ -44,6 +44,7 @@ promise.save() # so that we can restore the GroupResult in tests except (NotImplementedError, AttributeError): self.log.info('Unable to call save_group with current result backend.') + # FIXME: what to do in terms of return here? return promise.id diff --git a/swh/lister/cgit/lister.py b/swh/lister/cgit/lister.py --- a/swh/lister/cgit/lister.py +++ b/swh/lister/cgit/lister.py @@ -78,6 +78,7 @@ } def run(self): + status = 'uneventful' total = 0 for repos in grouper(self.get_repos(), 10): models = list(filter(None, (self.build_model(repo) @@ -87,6 +88,9 @@ self.db_session.commit() total += len(injected_repos) logger.debug('Scheduled %s tasks for %s', total, self.url) + status = 'eventful' + + return {'status': status} def get_repos(self): """Generate git 'project' URLs found on the current CGit server diff --git a/swh/lister/cgit/tasks.py b/swh/lister/cgit/tasks.py --- a/swh/lister/cgit/tasks.py +++ b/swh/lister/cgit/tasks.py @@ -10,7 +10,7 @@ @shared_task(name=__name__ + '.CGitListerTask') def list_cgit(**lister_args): '''Lister task for CGit instances''' - CGitLister(**lister_args).run() + return CGitLister(**lister_args).run() @shared_task(name=__name__ + '.ping') diff --git a/swh/lister/core/indexing_lister.py b/swh/lister/core/indexing_lister.py --- a/swh/lister/core/indexing_lister.py +++ b/swh/lister/core/indexing_lister.py @@ -213,6 +213,7 @@ Returns: nothing """ + status = 'uneventful' self.min_index = min_bound self.max_index = max_bound @@ -243,9 +244,11 @@ logger.debug('Flushing updates at index %s', i) self.db_session.commit() self.db_session = self.mk_session() + status = 'eventful' self.db_session.commit() self.db_session = self.mk_session() + return {'status': status} class IndexingHttpLister(ListerHttpTransport, IndexingLister): diff --git a/swh/lister/core/page_by_page_lister.py b/swh/lister/core/page_by_page_lister.py --- a/swh/lister/core/page_by_page_lister.py +++ b/swh/lister/core/page_by_page_lister.py @@ -110,6 +110,7 @@ nothing """ + status = 'uneventful' page = min_bound or 0 loop_count = 0 @@ -127,6 +128,7 @@ elif not injected_repos: logging.info('Repositories already seen, stopping') break + status = 'eventful' next_page = self.get_next_target_from_response(response) @@ -149,6 +151,8 @@ self.db_session.commit() self.db_session = self.mk_session() + return {'status': status} + class PageByPageHttpLister(ListerHttpTransport, PageByPageLister): """Convenience class for ensuring right lookup and init order when diff --git a/swh/lister/core/simple_lister.py b/swh/lister/core/simple_lister.py --- a/swh/lister/core/simple_lister.py +++ b/swh/lister/core/simple_lister.py @@ -89,3 +89,8 @@ response, injected_repos = self.ingest_data(dump_not_used_identifier) if not response and not injected_repos: logging.info('No response from api server, stopping') + status = 'uneventful' + else: + status = 'eventful' + + return {'status': status} diff --git a/swh/lister/cran/tasks.py b/swh/lister/cran/tasks.py --- a/swh/lister/cran/tasks.py +++ b/swh/lister/cran/tasks.py @@ -10,7 +10,7 @@ @shared_task(name=__name__ + '.CRANListerTask') def list_cran(**lister_args): '''Lister task for the CRAN registry''' - CRANLister(**lister_args).run() + return CRANLister(**lister_args).run() @shared_task(name=__name__ + '.ping') diff --git a/swh/lister/debian/lister.py b/swh/lister/debian/lister.py --- a/swh/lister/debian/lister.py +++ b/swh/lister/debian/lister.py @@ -216,12 +216,14 @@ .one_or_none() if not distribution: - raise ValueError("Distribution %s is not registered" % - self.distribution) + logger.error("Distribution %s is not registered" % + self.distribution) + return {'status': 'failed'} if not distribution.type == 'deb': - raise ValueError("Distribution %s is not a Debian derivative" % - distribution) + logger.error("Distribution %s is not a Debian derivative" % + distribution) + return {'status': 'failed'} date = self.date @@ -250,4 +252,4 @@ self.db_session.commit() - return True + return {'status': 'eventful'} diff --git a/swh/lister/debian/tasks.py b/swh/lister/debian/tasks.py --- a/swh/lister/debian/tasks.py +++ b/swh/lister/debian/tasks.py @@ -10,7 +10,7 @@ @shared_task(name=__name__ + '.DebianListerTask') def list_debian_distribution(distribution, **lister_args): '''List a Debian distribution''' - DebianLister(distribution=distribution, **lister_args).run() + return DebianLister(distribution=distribution, **lister_args).run() @shared_task(name=__name__ + '.ping') diff --git a/swh/lister/github/tasks.py b/swh/lister/github/tasks.py --- a/swh/lister/github/tasks.py +++ b/swh/lister/github/tasks.py @@ -15,13 +15,13 @@ def list_github_incremental(**lister_args): 'Incremental update of GitHub' lister = GitHubLister(**lister_args) - lister.run(min_bound=lister.db_last_index(), max_bound=None) + return lister.run(min_bound=lister.db_last_index(), max_bound=None) @shared_task(name=__name__ + '.RangeGitHubLister') def _range_github_lister(start, end, **lister_args): lister = GitHubLister(**lister_args) - lister.run(min_bound=start, max_bound=end) + return lister.run(min_bound=start, max_bound=end) @shared_task(name=__name__ + '.FullGitHubRelister', bind=True) @@ -44,6 +44,7 @@ promise.save() # so that we can restore the GroupResult in tests except (NotImplementedError, AttributeError): self.log.info('Unable to call save_group with current result backend.') + # FIXME: what to do in terms of return here? return promise.id diff --git a/swh/lister/gitlab/tasks.py b/swh/lister/gitlab/tasks.py --- a/swh/lister/gitlab/tasks.py +++ b/swh/lister/gitlab/tasks.py @@ -20,13 +20,13 @@ lister = GitLabLister(**lister_args) total_pages = lister.get_pages_information()[1] # stopping as soon as existing origins for that instance are detected - lister.run(min_bound=1, max_bound=total_pages, check_existence=True) + return lister.run(min_bound=1, max_bound=total_pages, check_existence=True) @shared_task(name=__name__ + '.RangeGitLabLister') def _range_gitlab_lister(start, end, **lister_args): lister = GitLabLister(**lister_args) - lister.run(min_bound=start, max_bound=end) + return lister.run(min_bound=start, max_bound=end) @shared_task(name=__name__ + '.FullGitLabRelister', bind=True) @@ -43,6 +43,7 @@ promise.save() except (NotImplementedError, AttributeError): self.log.info('Unable to call save_group with current result backend.') + # FIXME: what to do in terms of return here? return promise.id diff --git a/swh/lister/gnu/tasks.py b/swh/lister/gnu/tasks.py --- a/swh/lister/gnu/tasks.py +++ b/swh/lister/gnu/tasks.py @@ -9,8 +9,8 @@ @shared_task(name=__name__ + '.GNUListerTask') def list_gnu_full(**lister_args): - 'List lister for the GNU source code archive' - GNULister(**lister_args).run() + """List lister for the GNU source code archive""" + return GNULister(**lister_args).run() @shared_task(name=__name__ + '.ping') diff --git a/swh/lister/npm/tasks.py b/swh/lister/npm/tasks.py --- a/swh/lister/npm/tasks.py +++ b/swh/lister/npm/tasks.py @@ -45,7 +45,7 @@ 'Full lister for the npm (javascript) registry' lister = NpmLister(**lister_args) with save_registry_state(lister): - lister.run() + return lister.run() @shared_task(name=__name__ + '.NpmIncrementalListerTask') @@ -54,7 +54,7 @@ lister = NpmIncrementalLister(**lister_args) update_seq_start = get_last_update_seq(lister) with save_registry_state(lister): - lister.run(min_bound=update_seq_start) + return lister.run(min_bound=update_seq_start) @shared_task(name=__name__ + '.ping') diff --git a/swh/lister/phabricator/tasks.py b/swh/lister/phabricator/tasks.py --- a/swh/lister/phabricator/tasks.py +++ b/swh/lister/phabricator/tasks.py @@ -9,8 +9,8 @@ @shared_task(name=__name__ + '.FullPhabricatorLister') def list_phabricator_full(**lister_args): - 'Full update of a Phabricator instance' - PhabricatorLister(**lister_args).run() + """Full update of a Phabricator instance""" + return PhabricatorLister(**lister_args).run() @shared_task(name=__name__ + '.ping') diff --git a/swh/lister/pypi/tasks.py b/swh/lister/pypi/tasks.py --- a/swh/lister/pypi/tasks.py +++ b/swh/lister/pypi/tasks.py @@ -10,7 +10,7 @@ @shared_task(name=__name__ + '.PyPIListerTask') def list_pypi(**lister_args): 'Full update of the PyPI (python) registry' - PyPILister(**lister_args).run() + return PyPILister(**lister_args).run() @shared_task(name=__name__ + '.ping')