Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/loader.py
| Show All 37 Lines | class PackageLoader: | ||||
| # Origin visit type (str) set by the loader | # Origin visit type (str) set by the loader | ||||
| visit_type = '' | visit_type = '' | ||||
| def __init__(self, url): | def __init__(self, url): | ||||
| """Loader's constructor. This raises exception if the minimal required | """Loader's constructor. This raises exception if the minimal required | ||||
| configuration is missing (cf. fn:`check` method). | configuration is missing (cf. fn:`check` method). | ||||
| Args: | Args: | ||||
| url (str): Origin url to load data from | url (str): Origin url to load data from | ||||
ardumont: Please move the type in the constructor. | |||||
| """ | """ | ||||
| # This expects to use the environment variable SWH_CONFIG_FILENAME | # This expects to use the environment variable SWH_CONFIG_FILENAME | ||||
| self.config = SWHConfig.parse_config_file() | self.config = SWHConfig.parse_config_file() | ||||
| self._check_configuration() | self._check_configuration() | ||||
| self.storage = get_storage(**self.config['storage']) | self.storage = get_storage(**self.config['storage']) | ||||
| self.url = url | self.url = url | ||||
| self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc) | self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc) | ||||
| ▲ Show 20 Lines • Show All 301 Lines • ▼ Show 20 Lines | def _load_revision(self, p_info, origin) -> Tuple[Optional[Sha1Git], bool]: | ||||
| with tempfile.TemporaryDirectory() as tmpdir: | with tempfile.TemporaryDirectory() as tmpdir: | ||||
| try: | try: | ||||
| dl_artifacts = self.download_package(p_info, tmpdir) | dl_artifacts = self.download_package(p_info, tmpdir) | ||||
| except Exception: | except Exception: | ||||
| logger.exception('Unable to retrieve %s', | logger.exception('Unable to retrieve %s', | ||||
| p_info) | p_info) | ||||
| return (None, False) | return (None, False) | ||||
| try: | |||||
| uncompressed_path = self.uncompress(dl_artifacts, dest=tmpdir) | uncompressed_path = self.uncompress(dl_artifacts, dest=tmpdir) | ||||
| logger.debug('uncompressed_path: %s', uncompressed_path) | logger.debug('uncompressed_path: %s', uncompressed_path) | ||||
| except ValueError: | |||||
| logger.exception('Fail to uncompress %s', | |||||
| p_info['url']) | |||||
| return (None, False) | |||||
| directory = from_disk.Directory.from_disk( | directory = from_disk.Directory.from_disk( | ||||
| path=uncompressed_path.encode('utf-8'), | path=uncompressed_path.encode('utf-8'), | ||||
| max_content_length=self.max_content_size) | max_content_length=self.max_content_size) | ||||
| contents: List[Content] = [] | contents: List[Content] = [] | ||||
| skipped_contents: List[SkippedContent] = [] | skipped_contents: List[SkippedContent] = [] | ||||
| directories: List[Directory] = [] | directories: List[Directory] = [] | ||||
| ▲ Show 20 Lines • Show All 46 Lines • Show Last 20 Lines | |||||
Please move the type in the constructor.