Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/core/simple_lister.py
| Show All 39 Lines | def ingest_data(self, identifier, checks=False): | ||||
| """ | """ | ||||
| response = self.safely_issue_request(identifier) | response = self.safely_issue_request(identifier) | ||||
| response = self.list_packages(response) | response = self.list_packages(response) | ||||
| if not response: | if not response: | ||||
| return response, [] | return response, [] | ||||
| models_list = self.transport_response_simplified(response) | models_list = self.transport_response_simplified(response) | ||||
| models_list = self.filter_before_inject(models_list) | models_list = self.filter_before_inject(models_list) | ||||
| all_injected = [] | all_injected = [] | ||||
| for models in utils.grouper(models_list, n=10000): | for models in utils.grouper(models_list, n=1000): | ||||
| models = list(models) | models = list(models) | ||||
| logging.debug('models: %s' % len(models)) | logging.debug('models: %s' % len(models)) | ||||
| # inject into local db | # inject into local db | ||||
| injected = self.inject_repo_data_into_db(models) | injected = self.inject_repo_data_into_db(models) | ||||
| # queue workers | # queue workers | ||||
| self.schedule_missing_tasks(models, injected) | self.schedule_missing_tasks(models, injected) | ||||
| all_injected.append(injected) | all_injected.append(injected) | ||||
| # flush | # flush | ||||
| Show All 18 Lines | |||||