diff --git a/swh/indexer/indexer.py b/swh/indexer/indexer.py --- a/swh/indexer/indexer.py +++ b/swh/indexer/indexer.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2017 The Software Heritage developers +# Copyright (C) 2016-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -14,7 +14,6 @@ from swh.objstorage import get_objstorage from swh.objstorage.exc import ObjNotFoundError from swh.model import hashutil -from swh.scheduler.utils import get_task from swh.indexer.storage import get_indexer_storage, INDEXER_CFG_KEY @@ -63,8 +62,7 @@ shutil.rmtree(temp_dir) -class BaseIndexer(SWHConfig, - metaclass=abc.ABCMeta): +class BaseIndexer(SWHConfig, metaclass=abc.ABCMeta): """Base class for indexers to inherit from. The main entry point is the :func:`run` function which is in @@ -127,10 +125,6 @@ 'url': 'http://localhost:5007/' } }), - - # queue to reschedule if problem (none for no rescheduling, - # the default) - 'rescheduling_task': ('str', None), 'storage': ('dict', { 'cls': 'remote', 'args': { @@ -200,11 +194,6 @@ self.objstorage = get_objstorage(objstorage['cls'], objstorage['args']) idx_storage = self.config[INDEXER_CFG_KEY] self.idx_storage = get_indexer_storage(**idx_storage) - rescheduling_task = self.config['rescheduling_task'] - if rescheduling_task: - self.rescheduling_task = get_task(rescheduling_task) - else: - self.rescheduling_task = None _log = logging.getLogger('requests.packages.urllib3.connectionpool') _log.setLevel(logging.WARN) @@ -381,9 +370,6 @@ except Exception: self.log.exception( 'Problem when reading contents metadata.') - if self.rescheduling_task: - self.log.warn('Rescheduling batch') - self.rescheduling_task.delay(ids, policy_update) class OriginIndexer(BaseIndexer): diff --git a/swh/indexer/rehash.py b/swh/indexer/rehash.py --- a/swh/indexer/rehash.py +++ b/swh/indexer/rehash.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017 The Software Heritage developers +# Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -14,7 +14,6 @@ from swh.objstorage import get_objstorage from swh.objstorage.exc import ObjNotFoundError from swh.storage import get_storage -from swh.scheduler.utils import get_task class RecomputeChecksums(SWHConfig): @@ -61,8 +60,6 @@ 'batch_size_retrieve_content': ('int', 10), # Number of contents to update at the same time 'batch_size_update': ('int', 100), - # Rescheduling task on error (if None, nothing is done) - 'rescheduling_task': ('str', None), } CONFIG_BASE_FILENAME = 'indexer/rehash' @@ -80,12 +77,6 @@ 'batch_size_update'] self.log = logging.getLogger('swh.indexer.rehash') - rescheduling_task = self.config['rescheduling_task'] - if rescheduling_task: - self.rescheduling_task = get_task(rescheduling_task) - else: - self.rescheduling_task = None - if not self.compute_checksums: raise ValueError('Checksums list should not be empty.') @@ -123,10 +114,6 @@ except Exception: self.log.exception( 'Problem when reading contents metadata.') - if self.rescheduling_task: - self.log.warn('Rescheduling batch.') - cs = [{'sha1': sha1} for sha1 in contents_iter[1]] - self.rescheduling_task.delay(cs) continue for content in content_metadata: @@ -182,8 +169,4 @@ keys=keys) except Exception: self.log.exception('Problem during update.') - if self.rescheduling_task: - self.log.warn('Rescheduling batch.') - cs = [{'sha1': c['sha1']} for c in contents] - self.rescheduling_task.delay(cs) continue diff --git a/swh/indexer/tests/test_language.py b/swh/indexer/tests/test_language.py --- a/swh/indexer/tests/test_language.py +++ b/swh/indexer/tests/test_language.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017 The Software Heritage developers +# Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -30,7 +30,6 @@ def prepare(self): self.config = { 'destination_task': None, - 'rescheduling_task': None, 'tools': { 'name': 'pygments', 'version': '2.0.1+dfsg-1.1+deb8u1', @@ -45,7 +44,6 @@ self.log = logging.getLogger('swh.indexer') self.objstorage = MockObjStorage() self.destination_task = None - self.rescheduling_task = self.config['rescheduling_task'] self.tool_config = self.config['tools']['configuration'] self.max_content_size = self.tool_config['max_content_size'] self.tools = self.register_tools(self.config['tools']) diff --git a/swh/indexer/tests/test_metadata.py b/swh/indexer/tests/test_metadata.py --- a/swh/indexer/tests/test_metadata.py +++ b/swh/indexer/tests/test_metadata.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017 The Software Heritage developers +# Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -20,14 +20,10 @@ indexing tests. """ def prepare(self): - self.config.update({ - 'rescheduling_task': None, - }) self.idx_storage = MockIndexerStorage() self.log = logging.getLogger('swh.indexer') self.objstorage = MockObjStorage() self.destination_task = None - self.rescheduling_task = self.config['rescheduling_task'] self.tools = self.register_tools(self.config['tools']) self.tool = self.tools[0] self.results = [] @@ -42,7 +38,6 @@ def prepare(self): self.config = { - 'rescheduling_task': None, 'storage': { 'cls': 'remote', 'args': { @@ -63,7 +58,6 @@ self.log = logging.getLogger('swh.indexer') self.objstorage = MockObjStorage() self.destination_task = None - self.rescheduling_task = self.config['rescheduling_task'] self.tools = self.register_tools(self.config['tools']) self.tool = self.tools[0] self.results = [] diff --git a/swh/indexer/tests/test_mimetype.py b/swh/indexer/tests/test_mimetype.py --- a/swh/indexer/tests/test_mimetype.py +++ b/swh/indexer/tests/test_mimetype.py @@ -33,7 +33,6 @@ def prepare(self): self.config = { 'destination_task': None, - 'rescheduling_task': None, 'tools': { 'name': 'file', 'version': '1:5.30-1+deb9u1', @@ -46,8 +45,6 @@ self.idx_storage = _MockIndexerStorage() self.log = logging.getLogger('swh.indexer') self.objstorage = MockObjStorage() - self.destination_task = None - self.rescheduling_task = self.config['rescheduling_task'] self.destination_task = self.config['destination_task'] self.tools = self.register_tools(self.config['tools']) self.tool = self.tools[0]