Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/indexer.py
# Copyright (C) 2016-2018 The Software Heritage developers | # Copyright (C) 2016-2018 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import abc | import abc | ||||
import ast | import ast | ||||
import os | import os | ||||
import logging | import logging | ||||
import shutil | import shutil | ||||
import tempfile | import tempfile | ||||
import datetime | import datetime | ||||
from copy import deepcopy | from copy import deepcopy | ||||
from contextlib import contextmanager | from contextlib import contextmanager | ||||
from swh.scheduler import get_scheduler | from swh.scheduler import get_scheduler | ||||
try: | |||||
from swh.scheduler import CONFIG as SWH_CONFIG | |||||
except ImportError: | |||||
# for swh-scheduler < 0.0.47 bw compat | |||||
SWH_CONFIG = None | |||||
from swh.storage import get_storage | from swh.storage import get_storage | ||||
from swh.core.config import SWHConfig | from swh.core.config import SWHConfig | ||||
from swh.objstorage import get_objstorage | from swh.objstorage import get_objstorage | ||||
from swh.objstorage.exc import ObjNotFoundError | from swh.objstorage.exc import ObjNotFoundError | ||||
from swh.indexer.storage import get_indexer_storage, INDEXER_CFG_KEY | from swh.indexer.storage import get_indexer_storage, INDEXER_CFG_KEY | ||||
from swh.model import hashutil | from swh.model import hashutil | ||||
from swh.core import utils | from swh.core import utils | ||||
▲ Show 20 Lines • Show All 104 Lines • ▼ Show 20 Lines | class BaseIndexer(SWHConfig, metaclass=abc.ABCMeta): | ||||
def __init__(self, config=None, **kw): | def __init__(self, config=None, **kw): | ||||
"""Prepare and check that the indexer is ready to run. | """Prepare and check that the indexer is ready to run. | ||||
""" | """ | ||||
super().__init__() | super().__init__() | ||||
if config is not None: | if config is not None: | ||||
self.config = config | self.config = config | ||||
elif SWH_CONFIG: | |||||
self.config = SWH_CONFIG.copy() | |||||
else: | else: | ||||
config_keys = ('base_filename', 'config_filename', | config_keys = ('base_filename', 'config_filename', | ||||
'additional_configs', 'global_config') | 'additional_configs', 'global_config') | ||||
config_args = {k: v for k, v in kw.items() if k in config_keys} | config_args = {k: v for k, v in kw.items() if k in config_keys} | ||||
if self.ADDITIONAL_CONFIG: | if self.ADDITIONAL_CONFIG: | ||||
config_args.setdefault('additional_configs', []).append( | config_args.setdefault('additional_configs', []).append( | ||||
self.ADDITIONAL_CONFIG) | self.ADDITIONAL_CONFIG) | ||||
self.config = self.parse_config_file(**config_args) | self.config = self.parse_config_file(**config_args) | ||||
▲ Show 20 Lines • Show All 506 Lines • Show Last 20 Lines |