diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py --- a/swh/lister/bitbucket/lister.py +++ b/swh/lister/bitbucket/lister.py @@ -6,7 +6,6 @@ import iso8601 from datetime import datetime - from urllib import parse from swh.lister.bitbucket.models import BitBucketModel @@ -15,7 +14,6 @@ logger = logging.getLogger(__name__) - DEFAULT_BITBUCKET_PAGE = 10 @@ -45,15 +43,25 @@ } def get_next_target_from_response(self, response): + """This will read the 'next' link from the api response if any + and return it as an isoformatted date string. + + Returns: + next date as an isoformatted date string + + """ body = response.json() - if 'next' in body: - return parse.unquote(body['next'].split('after=')[1]) + next_ = body.get('next') + if next_ is not None: + next_ = parse.urlparse(next_) + return parse.parse_qs(next_.query)['after'][0] def transport_response_simplified(self, response): repos = response.json()['values'] return [self.get_model_from_repo(repo) for repo in repos] def request_uri(self, identifier): + identifier = parse.quote(identifier) return super().request_uri(identifier or '1970-01-01') def is_within_bounds(self, inner, lower=None, upper=None): diff --git a/swh/lister/core/indexing_lister.py b/swh/lister/core/indexing_lister.py --- a/swh/lister/core/indexing_lister.py +++ b/swh/lister/core/indexing_lister.py @@ -217,17 +217,22 @@ self.disable_deleted_repo_tasks(index, next_index, keep_these) # termination condition - if next_index is None or next_index == index: - logger.info('stopping after index %s, no next link found' % + if next_index is None: + logger.info('No next link found, stopping after index %s', index) return + if next_index == index: + logger.warn( + 'Next link found same as current one %s', + next_index) + return index = next_index logger.debug('Index: %s', index) yield i for i in ingest_indexes(): if (i % 20) == 0: - logger.info('flushing updates at index %s', i) + logger.debug('Flushing updates at index %s', i) self.db_session.commit() self.db_session = self.mk_session()