diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py --- a/swh/lister/bitbucket/lister.py +++ b/swh/lister/bitbucket/lister.py @@ -30,6 +30,9 @@ if per_page != DEFAULT_BITBUCKET_PAGE: self.PATH_TEMPLATE = '%s&pagelen=%s' % ( self.PATH_TEMPLATE, per_page) + # to stay consistent with prior behavior (20 * 10 repositories then) + self.flush_packet_db = int( + (self.flush_packet_db * DEFAULT_BITBUCKET_PAGE) / per_page) def get_model_from_repo(self, repo): return { diff --git a/swh/lister/core/indexing_lister.py b/swh/lister/core/indexing_lister.py --- a/swh/lister/core/indexing_lister.py +++ b/swh/lister/core/indexing_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2017 the Software Heritage developers +# Copyright (C) 2015-2019 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -16,6 +16,7 @@ class IndexingLister(ListerBase): + flush_packet_db = 20 """Lister* intermediate class for any service that follows the pattern: - The service must report at least one stable unique identifier, known @@ -222,7 +223,7 @@ yield i for i in ingest_indexes(): - if (i % 20) == 0: + if (i % self.flush_packet_db) == 0: logger.debug('Flushing updates at index %s', i) self.db_session.commit() self.db_session = self.mk_session()