diff --git a/swh/indexer/indexer.py b/swh/indexer/indexer.py --- a/swh/indexer/indexer.py +++ b/swh/indexer/indexer.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2017 The Software Heritage developers +# Copyright (C) 2016-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -16,8 +16,6 @@ from swh.objstorage import get_objstorage from swh.objstorage.exc import ObjNotFoundError from swh.model import hashutil -from swh.scheduler.utils import get_task -from swh.scheduler import get_scheduler from swh.indexer.storage import get_indexer_storage, INDEXER_CFG_KEY @@ -66,8 +64,7 @@ shutil.rmtree(temp_dir) -class BaseIndexer(SWHConfig, - metaclass=abc.ABCMeta): +class BaseIndexer(SWHConfig, metaclass=abc.ABCMeta): """Base class for indexers to inherit from. The main entry point is the :func:`run` function which is in @@ -130,10 +127,6 @@ 'url': 'http://localhost:5007/' } }), - - # queue to reschedule if problem (none for no rescheduling, - # the default) - 'rescheduling_task': ('str', None), 'storage': ('dict', { 'cls': 'remote', 'args': { @@ -203,11 +196,6 @@ self.objstorage = get_objstorage(objstorage['cls'], objstorage['args']) idx_storage = self.config[INDEXER_CFG_KEY] self.idx_storage = get_indexer_storage(**idx_storage) - rescheduling_task = self.config['rescheduling_task'] - if rescheduling_task: - self.rescheduling_task = get_task(rescheduling_task) - else: - self.rescheduling_task = None _log = logging.getLogger('requests.packages.urllib3.connectionpool') _log.setLevel(logging.WARN) @@ -404,9 +392,6 @@ except Exception: self.log.exception( 'Problem when reading contents metadata.') - if self.rescheduling_task: - self.log.warn('Rescheduling batch') - self.rescheduling_task.delay(ids, policy_update) class OriginIndexer(BaseIndexer): diff --git a/swh/indexer/tests/test_language.py b/swh/indexer/tests/test_language.py --- a/swh/indexer/tests/test_language.py +++ b/swh/indexer/tests/test_language.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017 The Software Heritage developers +# Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -30,7 +30,6 @@ def prepare(self): self.config = { 'destination_task': None, - 'rescheduling_task': None, 'tools': { 'name': 'pygments', 'version': '2.0.1+dfsg-1.1+deb8u1', @@ -45,7 +44,6 @@ self.log = logging.getLogger('swh.indexer') self.objstorage = MockObjStorage() self.destination_task = None - self.rescheduling_task = self.config['rescheduling_task'] self.tool_config = self.config['tools']['configuration'] self.max_content_size = self.tool_config['max_content_size'] self.tools = self.register_tools(self.config['tools']) diff --git a/swh/indexer/tests/test_metadata.py b/swh/indexer/tests/test_metadata.py --- a/swh/indexer/tests/test_metadata.py +++ b/swh/indexer/tests/test_metadata.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017 The Software Heritage developers +# Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -20,14 +20,10 @@ indexing tests. """ def prepare(self): - self.config.update({ - 'rescheduling_task': None, - }) self.idx_storage = MockIndexerStorage() self.log = logging.getLogger('swh.indexer') self.objstorage = MockObjStorage() self.destination_task = None - self.rescheduling_task = self.config['rescheduling_task'] self.tools = self.register_tools(self.config['tools']) self.tool = self.tools[0] self.results = [] @@ -42,7 +38,6 @@ def prepare(self): self.config = { - 'rescheduling_task': None, 'storage': { 'cls': 'remote', 'args': { @@ -63,7 +58,6 @@ self.log = logging.getLogger('swh.indexer') self.objstorage = MockObjStorage() self.destination_task = None - self.rescheduling_task = self.config['rescheduling_task'] self.tools = self.register_tools(self.config['tools']) self.tool = self.tools[0] self.results = [] diff --git a/swh/indexer/tests/test_mimetype.py b/swh/indexer/tests/test_mimetype.py --- a/swh/indexer/tests/test_mimetype.py +++ b/swh/indexer/tests/test_mimetype.py @@ -33,7 +33,6 @@ def prepare(self): self.config = { 'destination_task': None, - 'rescheduling_task': None, 'tools': { 'name': 'file', 'version': '1:5.30-1+deb9u1', @@ -46,8 +45,6 @@ self.idx_storage = _MockIndexerStorage() self.log = logging.getLogger('swh.indexer') self.objstorage = MockObjStorage() - self.destination_task = None - self.rescheduling_task = self.config['rescheduling_task'] self.destination_task = self.config['destination_task'] self.tools = self.register_tools(self.config['tools']) self.tool = self.tools[0]