diff --git a/swh/lister/cgit/lister.py b/swh/lister/cgit/lister.py --- a/swh/lister/cgit/lister.py +++ b/swh/lister/cgit/lister.py @@ -3,6 +3,7 @@ # See top-level LICENSE file for more information import re +import logging from urllib.parse import urlparse, urljoin from bs4 import BeautifulSoup @@ -15,6 +16,9 @@ from swh.lister.core.lister_base import ListerBase +logger = logging.getLogger(__name__) + + class CGitLister(ListerBase): """Lister class for CGit repositories. @@ -70,12 +74,15 @@ self.session.mount(self.url, HTTPAdapter(max_retries=3)) def run(self): - for repos in grouper(self.get_repos(), 100): + total = 0 + for repos in grouper(self.get_repos(), 10): models = list(filter(None, (self.build_model(repo) for repo in repos))) injected_repos = self.inject_repo_data_into_db(models) self.schedule_missing_tasks(models, injected_repos) self.db_session.commit() + total += len(injected_repos) + logger.debug('Scheduled %s tasks for %s', total, self.url) def get_repos(self): """Generate git 'project' URLs found on the current CGit server