Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/loader.py
| Show First 20 Lines • Show All 63 Lines • ▼ Show 20 Lines | def __init__(self): | ||||
| } | } | ||||
| def pre_cleanup(self): | def pre_cleanup(self): | ||||
| """To prevent disk explosion if some other workers exploded | """To prevent disk explosion if some other workers exploded | ||||
| in mid-air (OOM killed), we try and clean up dangling files. | in mid-air (OOM killed), we try and clean up dangling files. | ||||
| """ | """ | ||||
| if self.debug: | if self.debug: | ||||
| self.log.warn('%s Will not pre-clean up temp dir %s' % ( | self.log.warning('%s Will not pre-clean up temp dir %s' % ( | ||||
| DEBUG_MODE, self.temp_directory | DEBUG_MODE, self.temp_directory | ||||
| )) | )) | ||||
| return | return | ||||
| clean_dangling_folders(self.temp_directory, | clean_dangling_folders(self.temp_directory, | ||||
| pattern_check=self.TEMPORARY_DIR_PREFIX_PATTERN, | pattern_check=self.TEMPORARY_DIR_PREFIX_PATTERN, | ||||
| log=self.log) | log=self.log) | ||||
| def prepare_origin_visit(self, name, origin_url, **kwargs): | def prepare_origin_visit(self, name, origin_url, **kwargs): | ||||
| Show All 28 Lines | def prepare(self, name, origin_url, **kwargs): | ||||
| """ | """ | ||||
| self.package_contents = [] | self.package_contents = [] | ||||
| self.package_directories = [] | self.package_directories = [] | ||||
| self.package_revisions = [] | self.package_revisions = [] | ||||
| self.all_version_data = [] | self.all_version_data = [] | ||||
| self.latest_timestamp = 0 | self.latest_timestamp = 0 | ||||
| # Conceled the data into one dictionary to eleminate the need of | # Conceled the data into one dictionary to eleminate the need of | ||||
| # passing all the paramaters when required in some method | # passing all the parameters when required in some method | ||||
| self.package_details = { | self.package_details = { | ||||
| 'name': name, | 'name': name, | ||||
| 'origin_url': origin_url, | 'origin_url': origin_url, | ||||
| 'tarballs': kwargs['tarballs'], | 'tarballs': kwargs['tarballs'], | ||||
| } | } | ||||
| self.package_temp_dir = os.path.join(self.temp_directory, | self.package_temp_dir = os.path.join(self.temp_directory, | ||||
| self.package_details['name']) | self.package_details['name']) | ||||
| Show All 31 Lines | def prepare_package_versions(self, tarballs): | ||||
| """ | """ | ||||
| for package_version_data in tarballs: | for package_version_data in tarballs: | ||||
| tarball_url = package_version_data['archive'] | tarball_url = package_version_data['archive'] | ||||
| tarball_request = self._request(tarball_url, | tarball_request = self._request(tarball_url, | ||||
| throw_error=False) | throw_error=False) | ||||
| if tarball_request.status_code == 404: | if tarball_request.status_code == 404: | ||||
| self.log.warn('Tarball url %s returns a 404 error.', | self.log.warning('Tarball url %s returns a 404 error.', | ||||
| tarball_url) | tarball_url) | ||||
| self._visit_status = 'partial' | self._visit_status = 'partial' | ||||
| # FIX ME: Do we need to mark it `partial` here | # FIX ME: Do we need to mark it `partial` here | ||||
| continue | continue | ||||
| yield self._prepare_package_version(package_version_data, | yield self._prepare_package_version(package_version_data, | ||||
| tarball_request) | tarball_request) | ||||
| def _request(self, url, throw_error=True): | def _request(self, url, throw_error=True): | ||||
| ▲ Show 20 Lines • Show All 288 Lines • ▼ Show 20 Lines | def visit_status(self): | ||||
| return self._visit_status | return self._visit_status | ||||
| def cleanup(self): | def cleanup(self): | ||||
| """Clean up temporary disk use after downloading and extracting | """Clean up temporary disk use after downloading and extracting | ||||
| package tarballs. | package tarballs. | ||||
| """ | """ | ||||
| if self.debug: | if self.debug: | ||||
| self.log.warn('%s Will not clean up temp dir %s' % ( | self.log.warning('%s Will not clean up temp dir %s' % ( | ||||
| DEBUG_MODE, self.temp_directory | DEBUG_MODE, self.temp_directory | ||||
| )) | )) | ||||
| return | return | ||||
| if os.path.exists(self.temp_directory): | if os.path.exists(self.temp_directory): | ||||
| self.log.debug('Clean up %s' % self.temp_directory) | self.log.debug('Clean up %s' % self.temp_directory) | ||||
| shutil.rmtree(self.temp_directory) | shutil.rmtree(self.temp_directory) | ||||