diff --git a/swh/lister/core/lister_base.py b/swh/lister/core/lister_base.py --- a/swh/lister/core/lister_base.py +++ b/swh/lister/core/lister_base.py @@ -135,29 +135,26 @@ pass def filter_before_inject(self, models_list): - """Function run after transport_response_simplified but before - injection into the local db and creation of workers. Can be - used to eliminate some of the results if necessary. + """Filter models_list entries prior to injection in the db. + This is ran directly after `transport_response_simplified`. - MAY BE OVERRIDDEN if an intermediate Lister class needs to filter - results before injection without requiring every child class to do so. + Default implementation is to have no filtering. Args: models_list: list of dicts returned by transport_response_simplified. Returns: models_list with entries changed according to custom logic. + """ return models_list def do_additional_checks(self, models_list): - """Execute some additional checks on the model list. For example, to - check for existing repositories in the db. - - MAY BE OVERRIDDEN if an intermediate Lister class needs to - check some more the results before injection. + """Execute some additional checks on the model list (after the + filtering). - Checks are fine by default, returns the models_list as is by default. + Default implementation is to run no check at all and to return + the input as is. Args: models_list: list of dicts returned by @@ -384,20 +381,6 @@ return sql_repo - def origin_dict(self, origin_type, origin_url, **kwargs): - """Return special dict format for the origins list - - Args: - origin_type (string) - origin_url (string) - Returns: - the same information in a different form - """ - return { - 'type': origin_type, - 'url': origin_url, - } - def task_dict(self, origin_type, origin_url, **kwargs): """Return special dict format for the tasks list @@ -452,45 +435,34 @@ injected_repos[m['uid']] = self.db_inject_repo(m) return injected_repos - def create_missing_origins_and_tasks(self, models_list, injected_repos): - """Find any newly created db entries that don't yet have tasks or - origin objects assigned. + def schedule_missing_tasks(self, models_list, injected_repos): + """Find any newly created db entries that do not have been scheduled + yet. Args: - models_list: a list of dicts mapping keys in the db model for - each repo - injected_repos: dict of uid:sql_repo pairs that have just - been created + models_list ([Model]): List of dicts mapping keys in the db model + for each repo + injected_repos ([dict]): Dict of uid:sql_repo pairs that have just + been created + Returns: Nothing. Modifies injected_repos. + """ - origins = {} tasks = {} - def _origin_key(m): - _type = m.get('origin_type', m.get('type')) - _url = m.get('origin_url', m.get('url')) - return '%s-%s' % (_type, _url) - def _task_key(m): - return '%s-%s' % (m['type'], - json.dumps(m['arguments'], sort_keys=True)) + return '%s-%s' % ( + m['type'], + json.dumps(m['arguments'], sort_keys=True) + ) for m in models_list: ir = injected_repos[m['uid']] - if not ir.origin_id: - origin_dict = self.origin_dict(**m) - origins[_origin_key(m)] = (ir, m, origin_dict) if not ir.task_id: task_dict = self.task_dict(**m) tasks[_task_key(task_dict)] = (ir, m, task_dict) - new_origins = self.storage.origin_add( - (origin_dicts for (_, _, origin_dicts) in origins.values())) - for origin in new_origins: - ir, m, _ = origins[_origin_key(origin)] - ir.origin_id = origin['id'] - new_tasks = self.scheduler.create_tasks( (task_dicts for (_, _, task_dicts) in tasks.values())) for task in new_tasks: @@ -519,7 +491,7 @@ # inject into local db injected = self.inject_repo_data_into_db(models_list) # queue workers - self.create_missing_origins_and_tasks(models_list, injected) + self.schedule_missing_tasks(models_list, injected) return response, injected def save_response(self, response): diff --git a/swh/lister/core/simple_lister.py b/swh/lister/core/simple_lister.py --- a/swh/lister/core/simple_lister.py +++ b/swh/lister/core/simple_lister.py @@ -51,7 +51,7 @@ # inject into local db injected = self.inject_repo_data_into_db(models) # queue workers - self.create_missing_origins_and_tasks(models, injected) + self.schedule_missing_tasks(models, injected) all_injected.append(injected) # flush self.db_session.commit() diff --git a/swh/lister/core/tests/test_lister.py b/swh/lister/core/tests/test_lister.py --- a/swh/lister/core/tests/test_lister.py +++ b/swh/lister/core/tests/test_lister.py @@ -164,8 +164,8 @@ if k not in ['last_seen', 'task_id', 'origin_id', 'id']: self.assertIn(k, di) - def disable_storage_and_scheduler(self, fl): - fl.create_missing_origins_and_tasks = Mock(return_value=None) + def disable_scheduler(self, fl): + fl.schedule_missing_tasks = Mock(return_value=None) def disable_db(self, fl): fl.winnow_models = Mock(return_value=[]) @@ -176,7 +176,7 @@ http_mocker.get(self.test_re, text=self.mock_response) fl = self.get_fl() - self.disable_storage_and_scheduler(fl) + self.disable_scheduler(fl) self.disable_db(fl) fl.run(min_bound=1, max_bound=1) # stores no results @@ -185,7 +185,7 @@ http_mocker.get(self.test_re, text=self.mock_response) fl = self.get_fl() - self.disable_storage_and_scheduler(fl) + self.disable_scheduler(fl) self.disable_db(fl) fl.run(min_bound=self.first_index, max_bound=self.first_index) @@ -194,7 +194,7 @@ http_mocker.get(self.test_re, text=self.mock_response) fl = self.get_fl() - self.disable_storage_and_scheduler(fl) + self.disable_scheduler(fl) self.disable_db(fl) fl.run(min_bound=self.first_index) @@ -222,7 +222,7 @@ }) self.init_db(db, fl.MODEL) - self.disable_storage_and_scheduler(fl) + self.disable_scheduler(fl) fl.run(min_bound=self.first_index) diff --git a/swh/lister/debian/lister.py b/swh/lister/debian/lister.py --- a/swh/lister/debian/lister.py +++ b/swh/lister/debian/lister.py @@ -119,7 +119,7 @@ """Generate the Package entries that didn't previously exist. Contrary to SWHListerBase, we don't actually insert the data in - database. `create_missing_origins_and_tasks` does it once we have the + database. `schedule_missing_tasks` does it once we have the origin and task identifiers. """ by_name_version = {} @@ -173,7 +173,7 @@ self.db_session.add_all(added_packages) return added_packages - def create_missing_origins_and_tasks(self, models_list, added_packages): + def schedule_missing_tasks(self, models_list, added_packages): """We create tasks at the end of the full snapshot processing""" return diff --git a/swh/lister/phabricator/lister.py b/swh/lister/phabricator/lister.py --- a/swh/lister/phabricator/lister.py +++ b/swh/lister/phabricator/lister.py @@ -97,7 +97,7 @@ self.max_index = models_list[0]['indexable'] models_list = self.filter_before_inject(models_list) injected = self.inject_repo_data_into_db(models_list) - self.create_missing_origins_and_tasks(models_list, injected) + self.schedule_missing_tasks(models_list, injected) return self.max_index def run(self, min_bound=None, max_bound=None):