Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/core/indexing_lister.py
| Show First 20 Lines • Show All 207 Lines • ▼ Show 20 Lines | def run(self, min_bound=None, max_bound=None): | ||||
| than the desired max_bound. | than the desired max_bound. | ||||
| Args: | Args: | ||||
| min_bound (indexable type): optional index to start from | min_bound (indexable type): optional index to start from | ||||
| max_bound (indexable type): optional index to stop at | max_bound (indexable type): optional index to stop at | ||||
| Returns: | Returns: | ||||
| nothing | nothing | ||||
| """ | """ | ||||
| status = 'uneventful' | |||||
| self.min_index = min_bound | self.min_index = min_bound | ||||
| self.max_index = max_bound | self.max_index = max_bound | ||||
| def ingest_indexes(): | def ingest_indexes(): | ||||
| index = min_bound or self.default_min_bound | index = min_bound or self.default_min_bound | ||||
| for i in count(1): | for i in count(1): | ||||
| response, injected_repos = self.ingest_data(index) | response, injected_repos = self.ingest_data(index) | ||||
| if not response and not injected_repos: | if not response and not injected_repos: | ||||
| Show All 14 Lines | def run(self, min_bound=None, max_bound=None): | ||||
| logger.debug('Index: %s', index) | logger.debug('Index: %s', index) | ||||
| yield i | yield i | ||||
| for i in ingest_indexes(): | for i in ingest_indexes(): | ||||
| if (i % self.flush_packet_db) == 0: | if (i % self.flush_packet_db) == 0: | ||||
| logger.debug('Flushing updates at index %s', i) | logger.debug('Flushing updates at index %s', i) | ||||
| self.db_session.commit() | self.db_session.commit() | ||||
| self.db_session = self.mk_session() | self.db_session = self.mk_session() | ||||
| status = 'eventful' | |||||
| self.db_session.commit() | self.db_session.commit() | ||||
| self.db_session = self.mk_session() | self.db_session = self.mk_session() | ||||
| return {'status': status} | |||||
| class IndexingHttpLister(ListerHttpTransport, IndexingLister): | class IndexingHttpLister(ListerHttpTransport, IndexingLister): | ||||
| """Convenience class for ensuring right lookup and init order | """Convenience class for ensuring right lookup and init order | ||||
| when combining IndexingLister and ListerHttpTransport.""" | when combining IndexingLister and ListerHttpTransport.""" | ||||
| def __init__(self, url=None, override_config=None): | def __init__(self, url=None, override_config=None): | ||||
| IndexingLister.__init__(self, override_config=override_config) | IndexingLister.__init__(self, override_config=override_config) | ||||
| ListerHttpTransport.__init__(self, url=url) | ListerHttpTransport.__init__(self, url=url) | ||||