Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/cgit/lister.py
Show First 20 Lines • Show All 72 Lines • ▼ Show 20 Lines | def __init__(self, url=None, instance=None, override_config=None): | ||||
self.instance = instance | self.instance = instance | ||||
self.session = Session() | self.session = Session() | ||||
self.session.mount(self.url, HTTPAdapter(max_retries=3)) | self.session.mount(self.url, HTTPAdapter(max_retries=3)) | ||||
self.session.headers = { | self.session.headers = { | ||||
'User-Agent': USER_AGENT, | 'User-Agent': USER_AGENT, | ||||
} | } | ||||
def run(self): | def run(self): | ||||
status = 'uneventful' | |||||
total = 0 | total = 0 | ||||
for repos in grouper(self.get_repos(), 10): | for repos in grouper(self.get_repos(), 10): | ||||
models = list(filter(None, (self.build_model(repo) | models = list(filter(None, (self.build_model(repo) | ||||
for repo in repos))) | for repo in repos))) | ||||
injected_repos = self.inject_repo_data_into_db(models) | injected_repos = self.inject_repo_data_into_db(models) | ||||
self.schedule_missing_tasks(models, injected_repos) | self.schedule_missing_tasks(models, injected_repos) | ||||
self.db_session.commit() | self.db_session.commit() | ||||
total += len(injected_repos) | total += len(injected_repos) | ||||
logger.debug('Scheduled %s tasks for %s', total, self.url) | logger.debug('Scheduled %s tasks for %s', total, self.url) | ||||
status = 'eventful' | |||||
return {'status': status} | |||||
def get_repos(self): | def get_repos(self): | ||||
"""Generate git 'project' URLs found on the current CGit server | """Generate git 'project' URLs found on the current CGit server | ||||
""" | """ | ||||
next_page = self.url | next_page = self.url | ||||
while next_page: | while next_page: | ||||
bs_idx = self.get_and_parse(next_page) | bs_idx = self.get_and_parse(next_page) | ||||
▲ Show 20 Lines • Show All 45 Lines • Show Last 20 Lines |