Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/npm/loader.py
| Show First 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | def __init__(self): | ||||
| self.npm_client = NpmClient(self.temp_directory, self.log) | self.npm_client = NpmClient(self.temp_directory, self.log) | ||||
| def pre_cleanup(self): | def pre_cleanup(self): | ||||
| """ | """ | ||||
| To prevent disk explosion if some other workers exploded | To prevent disk explosion if some other workers exploded | ||||
| in mid-air (OOM killed), we try and clean up dangling files. | in mid-air (OOM killed), we try and clean up dangling files. | ||||
| """ | """ | ||||
| if self.debug: | if self.debug: | ||||
| self.log.warning('DEBUG: will not pre-clean up temp dir %s' % | self.log.warning('DEBUG: will not pre-clean up temp dir %s', | ||||
| self.temp_directory) | self.temp_directory) | ||||
| return | return | ||||
| clean_dangling_folders(self.config['temp_directory'], | clean_dangling_folders(self.config['temp_directory'], | ||||
| pattern_check=TEMPORARY_DIR_PREFIX_PATTERN, | pattern_check=TEMPORARY_DIR_PREFIX_PATTERN, | ||||
| log=self.log) | log=self.log) | ||||
| def cleanup(self): | def cleanup(self): | ||||
| """ | """ | ||||
| Clean up temporary disk use after downloading and extracting | Clean up temporary disk use after downloading and extracting | ||||
| npm source package tarballs. | npm source package tarballs. | ||||
| """ | """ | ||||
| if self.debug: | if self.debug: | ||||
| self.log.warning('DEBUG: will not clean up temp dir %s' % | self.log.warning('DEBUG: will not clean up temp dir %s', | ||||
| self.temp_directory) | self.temp_directory) | ||||
| return | return | ||||
| if os.path.exists(self.temp_directory): | if os.path.exists(self.temp_directory): | ||||
| self.log.debug('Clean up %s' % self.temp_directory) | self.log.debug('Clean up %s', self.temp_directory) | ||||
| shutil.rmtree(self.temp_directory) | shutil.rmtree(self.temp_directory) | ||||
| def load(self, package_name, package_url=None, | def load(self, package_name, package_url=None, | ||||
| package_metadata_url=None): | package_metadata_url=None): | ||||
| """ | """ | ||||
| Loader entrypoint to ingest source tarballs for a npm package. | Loader entrypoint to ingest source tarballs for a npm package. | ||||
| Args: | Args: | ||||
| ▲ Show 20 Lines • Show All 154 Lines • ▼ Show 20 Lines | def fetch_data(self): | ||||
| } | } | ||||
| revision['id'] = identifier_to_bytes(revision_identifier(revision)) | revision['id'] = identifier_to_bytes(revision_identifier(revision)) | ||||
| self.package_revisions.append(revision) | self.package_revisions.append(revision) | ||||
| package_key = (package_source_data['version'], | package_key = (package_source_data['version'], | ||||
| package_source_data['sha1']) | package_source_data['sha1']) | ||||
| self.known_versions[package_key] = revision['id'] | self.known_versions[package_key] = revision['id'] | ||||
olasd: Same remark here | |||||
| self.log.debug('Removing unpacked package files at %s', dir_path) | |||||
| shutil.rmtree(dir_path) | |||||
| return not self.done | return not self.done | ||||
| def _target_from_version(self, version, sha1): | def _target_from_version(self, version, sha1): | ||||
| """ | """ | ||||
| Return revision information if any for a given package version. | Return revision information if any for a given package version. | ||||
| """ | """ | ||||
| target = self.known_versions.get((version, sha1)) | target = self.known_versions.get((version, sha1)) | ||||
| return { | return { | ||||
| ▲ Show 20 Lines • Show All 61 Lines • Show Last 20 Lines | |||||
Same remark here