Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/core/indexing_lister.py
| Show First 20 Lines • Show All 146 Lines • ▼ Show 20 Lines | def disable_deleted_repo_tasks(self, start, end, keep_these): | ||||
| ) | ) | ||||
| tasks_to_disable = [repo.task_id for repo in deleted_repos | tasks_to_disable = [repo.task_id for repo in deleted_repos | ||||
| if repo.task_id is not None] | if repo.task_id is not None] | ||||
| if tasks_to_disable: | if tasks_to_disable: | ||||
| self.scheduler.disable_tasks(tasks_to_disable) | self.scheduler.disable_tasks(tasks_to_disable) | ||||
| for repo in deleted_repos: | for repo in deleted_repos: | ||||
| repo.task_id = None | repo.task_id = None | ||||
| def run(self, min_index=None, max_index=None): | def run(self, min_bound=None, max_bound=None): | ||||
| """Main entry function. Sequentially fetches repository data | """Main entry function. Sequentially fetches repository data | ||||
| from the service according to the basic outline in the class | from the service according to the basic outline in the class | ||||
| docstring, continually fetching sublists until either there | docstring, continually fetching sublists until either there | ||||
| is no next index reference given or the given next index is greater | is no next index reference given or the given next index is greater | ||||
| than the desired max_index. | than the desired max_bound. | ||||
| Args: | Args: | ||||
| min_index (indexable type): optional index to start from | min_bound (indexable type): optional index to start from | ||||
| max_index (indexable type): optional index to stop at | max_bound (indexable type): optional index to stop at | ||||
| Returns: | Returns: | ||||
| nothing | nothing | ||||
| """ | """ | ||||
| index = min_index or '' | index = min_bound or '' | ||||
| loop_count = 0 | loop_count = 0 | ||||
| self.min_index = min_index | self.min_index = min_bound | ||||
| self.max_index = max_index | self.max_index = max_bound | ||||
| while self.is_within_bounds(index, self.min_index, self.max_index): | while self.is_within_bounds(index, self.min_index, self.max_index): | ||||
| logging.info('listing repos starting at %s' % index) | logging.info('listing repos starting at %s' % index) | ||||
| response, injected_repos = self.ingest_data(index) | response, injected_repos = self.ingest_data(index) | ||||
| next_index = self.get_next_target_from_response(response) | next_index = self.get_next_target_from_response(response) | ||||
| Show All 20 Lines | def run(self, min_bound=None, max_bound=None): | ||||
| self.db_session.commit() | self.db_session.commit() | ||||
| self.db_session = self.mk_session() | self.db_session = self.mk_session() | ||||
| class SWHIndexingHttpLister(SWHListerHttpTransport, SWHIndexingLister): | class SWHIndexingHttpLister(SWHListerHttpTransport, SWHIndexingLister): | ||||
| """Convenience class for ensuring right lookup and init order | """Convenience class for ensuring right lookup and init order | ||||
| when combining SWHIndexingLister and SWHListerHttpTransport.""" | when combining SWHIndexingLister and SWHListerHttpTransport.""" | ||||
| def __init__(self, lister_name=None, api_baseurl=None, | def __init__(self, api_baseurl=None, override_config=None): | ||||
| override_config=None): | |||||
| SWHListerHttpTransport.__init__(self, api_baseurl=api_baseurl) | SWHListerHttpTransport.__init__(self, api_baseurl=api_baseurl) | ||||
| SWHIndexingLister.__init__(self, lister_name=lister_name, | SWHIndexingLister.__init__(self, override_config=override_config) | ||||
| override_config=override_config) | |||||