Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/tar/loader.py
| Show First 20 Lines • Show All 122 Lines • ▼ Show 20 Lines | class BaseTarLoader(BufferedLoader): | ||||
| """ | """ | ||||
| CONFIG_BASE_FILENAME = 'loader/tar' | CONFIG_BASE_FILENAME = 'loader/tar' | ||||
| ADDITIONAL_CONFIG = { | ADDITIONAL_CONFIG = { | ||||
| 'working_dir': ('string', '/tmp'), | 'working_dir': ('string', '/tmp'), | ||||
| 'debug': ('bool', False), # NOT FOR PRODUCTION | 'debug': ('bool', False), # NOT FOR PRODUCTION | ||||
| } | } | ||||
| visit_type = 'tar' | |||||
| def __init__(self, logging_class='swh.loader.tar.TarLoader', config=None): | def __init__(self, logging_class='swh.loader.tar.TarLoader', config=None): | ||||
| super().__init__(logging_class=logging_class, config=config) | super().__init__(logging_class=logging_class, config=config) | ||||
| self.local_cache = None | self.local_cache = None | ||||
| self.dir_path = None | self.dir_path = None | ||||
| working_dir = self.config.get('working_dir', tempfile.gettempdir()) | working_dir = self.config.get('working_dir', tempfile.gettempdir()) | ||||
| os.makedirs(working_dir, exist_ok=True) | os.makedirs(working_dir, exist_ok=True) | ||||
| self.temp_directory = mkdtemp( | self.temp_directory = mkdtemp( | ||||
| suffix='-%s' % os.getpid(), | suffix='-%s' % os.getpid(), | ||||
| Show All 25 Lines | def prepare_origin_visit(self, *, origin, visit_date=None, **kwargs): | ||||
| origin (dict): Dict with keys {url, type} | origin (dict): Dict with keys {url, type} | ||||
| visit_date (str): Date representing the date of the | visit_date (str): Date representing the date of the | ||||
| visit. None by default will make it the current time | visit. None by default will make it the current time | ||||
| during the loading process. | during the loading process. | ||||
| """ | """ | ||||
| self.origin = origin | self.origin = origin | ||||
| if 'type' not in self.origin: # let the type flow if present | if 'type' not in self.origin: # let the type flow if present | ||||
| self.origin['type'] = 'tar' | self.origin['type'] = self.visit_type | ||||
| self.visit_date = visit_date | self.visit_date = visit_date | ||||
| def get_tarball_url_to_retrieve(self): | def get_tarball_url_to_retrieve(self): | ||||
| """Compute the tarball url to allow retrieval | """Compute the tarball url to allow retrieval | ||||
| """ | """ | ||||
| raise NotImplementedError() | raise NotImplementedError() | ||||
| ▲ Show 20 Lines • Show All 158 Lines • Show Last 20 Lines | |||||