Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/core/lister_base.py
| Show All 15 Lines | |||||
| from swh.core import config | from swh.core import config | ||||
| from swh.scheduler import get_scheduler, utils | from swh.scheduler import get_scheduler, utils | ||||
| from swh.storage import get_storage | from swh.storage import get_storage | ||||
| from .abstractattribute import AbstractAttribute | from .abstractattribute import AbstractAttribute | ||||
| logger = logging.getLogger(__name__) | |||||
| def utcnow(): | def utcnow(): | ||||
| return datetime.datetime.now(tz=datetime.timezone.utc) | return datetime.datetime.now(tz=datetime.timezone.utc) | ||||
| class FetchError(RuntimeError): | class FetchError(RuntimeError): | ||||
| def __init__(self, response): | def __init__(self, response): | ||||
| self.response = response | self.response = response | ||||
| ▲ Show 20 Lines • Show All 209 Lines • ▼ Show 20 Lines | def ADDITIONAL_CONFIG(self): # noqa: N802 | ||||
| } | } | ||||
| INITIAL_BACKOFF = 10 | INITIAL_BACKOFF = 10 | ||||
| MAX_RETRIES = 7 | MAX_RETRIES = 7 | ||||
| CONN_SLEEP = 10 | CONN_SLEEP = 10 | ||||
| def __init__(self, override_config=None): | def __init__(self, override_config=None): | ||||
| self.backoff = self.INITIAL_BACKOFF | self.backoff = self.INITIAL_BACKOFF | ||||
| logging.debug('Loading config from %s' % self.CONFIG_BASE_FILENAME) | |||||
| self.config = self.parse_config_file( | self.config = self.parse_config_file( | ||||
| base_filename=self.CONFIG_BASE_FILENAME, | base_filename=self.CONFIG_BASE_FILENAME, | ||||
| additional_configs=[self.ADDITIONAL_CONFIG] | additional_configs=[self.ADDITIONAL_CONFIG] | ||||
| ) | ) | ||||
| self.config['cache_dir'] = os.path.expanduser(self.config['cache_dir']) | self.config['cache_dir'] = os.path.expanduser(self.config['cache_dir']) | ||||
| if self.config['cache_responses']: | if self.config['cache_responses']: | ||||
| config.prepare_folders(self.config, 'cache_dir') | config.prepare_folders(self.config, 'cache_dir') | ||||
| if override_config: | if override_config: | ||||
| self.config.update(override_config) | self.config.update(override_config) | ||||
| logger.debug('%s CONFIG=%s' % (self, self.config)) | |||||
| self.storage = get_storage(**self.config['storage']) | self.storage = get_storage(**self.config['storage']) | ||||
| self.scheduler = get_scheduler(**self.config['scheduler']) | self.scheduler = get_scheduler(**self.config['scheduler']) | ||||
| self.db_engine = create_engine(self.config['lister_db_url']) | self.db_engine = create_engine(self.config['lister_db_url']) | ||||
| self.mk_session = sessionmaker(bind=self.db_engine) | self.mk_session = sessionmaker(bind=self.db_engine) | ||||
| self.db_session = self.mk_session() | self.db_session = self.mk_session() | ||||
| def reset_backoff(self): | def reset_backoff(self): | ||||
| """Reset exponential backoff timeout to initial level.""" | """Reset exponential backoff timeout to initial level.""" | ||||
| ▲ Show 20 Lines • Show All 275 Lines • Show Last 20 Lines | |||||