Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/loader.py
| Show First 20 Lines • Show All 68 Lines • ▼ Show 20 Lines | logger.warning('Retry loading a batch', exc_info=False, extra={ | ||||
| error, | error, | ||||
| error.__traceback__, | error.__traceback__, | ||||
| ), | ), | ||||
| }) | }) | ||||
| return True | return True | ||||
| class SWHLoader(config.SWHConfig, metaclass=ABCMeta): | class BufferedLoader(config.SWHConfig, metaclass=ABCMeta): | ||||
| """Mixin base class for loader. | """Mixin base class for loader. | ||||
| To use this class, you must: | To use this class, you must: | ||||
| - inherit from this class | - inherit from this class | ||||
| - and implement the @abstractmethod methods: | - and implement the @abstractmethod methods: | ||||
| - :func:`prepare`: First step executed by the loader to prepare some | - :func:`prepare`: First step executed by the loader to prepare some | ||||
| ▲ Show 20 Lines • Show All 824 Lines • ▼ Show 20 Lines | def load(self, *args, **kwargs): | ||||
| return {'status': 'failed'} | return {'status': 'failed'} | ||||
| finally: | finally: | ||||
| self.flush() | self.flush() | ||||
| self.cleanup() | self.cleanup() | ||||
| return self.load_status() | return self.load_status() | ||||
| class SWHStatelessLoader(SWHLoader): | class UnbufferedLoader(BufferedLoader): | ||||
| """This base class is a pattern for stateless loaders. | """This base class is a pattern for unbuffered loaders. | ||||
| Stateless loaders are able to load all the data in one go. For | UnbufferedLoader loaders are able to load all the data in one go. For | ||||
| example, the loader defined in swh-loader-git | example, the loader defined in swh-loader-git | ||||
| :class:`BulkUpdater`. | :class:`BulkUpdater`. | ||||
| For other loaders (stateful one, (e.g :class:`SWHSvnLoader`), | For other loaders (stateful one, (e.g :class:`SWHSvnLoader`), | ||||
| inherit directly from :class:`SWHLoader`. | inherit directly from :class:`BufferedLoader`. | ||||
| """ | """ | ||||
| ADDITIONAL_CONFIG = {} | ADDITIONAL_CONFIG = {} | ||||
| def __init__(self, logging_class=None, config=None): | def __init__(self, logging_class=None, config=None): | ||||
| super().__init__(logging_class=logging_class, config=config) | super().__init__(logging_class=logging_class, config=config) | ||||
| self.visit_date = None # possibly overridden in self.prepare method | self.visit_date = None # possibly overridden in self.prepare method | ||||
| ▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | def eventful(self): | ||||
| """Whether the load was eventful""" | """Whether the load was eventful""" | ||||
| raise NotImplementedError | raise NotImplementedError | ||||
| def save_data(self): | def save_data(self): | ||||
| """Save the data associated to the current load""" | """Save the data associated to the current load""" | ||||
| raise NotImplementedError | raise NotImplementedError | ||||
| def flush(self): | def flush(self): | ||||
| """Stateless loader does not flush since it has no state to flush. | """Unbuffered loader does not flush since it has no state to flush. | ||||
| """ | """ | ||||
| pass | pass | ||||
| def store_data(self): | def store_data(self): | ||||
| if self.config['save_data']: | if self.config['save_data']: | ||||
| self.save_data() | self.save_data() | ||||
| Show All 10 Lines | |||||