Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/svn/loader.py
| Show First 20 Lines • Show All 104 Lines • ▼ Show 20 Lines | def __init__(self, url, origin_url=None, visit_date=None, | ||||
| self._snapshot = None | self._snapshot = None | ||||
| self._last_revision = None | self._last_revision = None | ||||
| self._visit_status = 'full' | self._visit_status = 'full' | ||||
| self._load_status = 'uneventful' | self._load_status = 'uneventful' | ||||
| self.visit_date = visit_date | self.visit_date = visit_date | ||||
| self.destination_path = destination_path | self.destination_path = destination_path | ||||
| self.start_from_scratch = start_from_scratch | self.start_from_scratch = start_from_scratch | ||||
| self.swh_revision = swh_revision | self.swh_revision = swh_revision | ||||
| self.max_content_length = self.config['max_content_size'] | |||||
| def pre_cleanup(self): | def pre_cleanup(self): | ||||
| """Cleanup potential dangling files from prior runs (e.g. OOM killed | """Cleanup potential dangling files from prior runs (e.g. OOM killed | ||||
| tasks) | tasks) | ||||
| """ | """ | ||||
| clean_dangling_folders(self.temp_directory, | clean_dangling_folders(self.temp_directory, | ||||
| pattern_check=TEMPORARY_DIR_PREFIX_PATTERN, | pattern_check=TEMPORARY_DIR_PREFIX_PATTERN, | ||||
| Show All 22 Lines | def swh_revision_hash_tree_at_svn_revision(self, revision): | ||||
| The hash tree directory as bytes. | The hash tree directory as bytes. | ||||
| """ | """ | ||||
| local_dirname, local_url = self.svnrepo.export_temporary(revision) | local_dirname, local_url = self.svnrepo.export_temporary(revision) | ||||
| h = Directory.from_disk(path=local_url).hash | h = Directory.from_disk(path=local_url).hash | ||||
| self.svnrepo.clean_fs(local_dirname) | self.svnrepo.clean_fs(local_dirname) | ||||
| return h | return h | ||||
| def get_svn_repo(self, svn_url, local_dirname, origin_url): | |||||
| """Instantiates the needed svnrepo collaborator to permit reading svn | |||||
| repository. | |||||
| Args: | |||||
| svn_url (str): the svn repository url to read from | |||||
| local_dirname (str): the local path on disk to compute data | |||||
| origin_url (str): the corresponding origin url | |||||
| Returns: | |||||
| Instance of :mod:`swh.loader.svn.svn` clients | |||||
| """ | |||||
| return svn.SvnRepo(svn_url, | |||||
| local_dirname=local_dirname, origin_url=origin_url) | |||||
| def swh_latest_snapshot_revision(self, origin_url, | def swh_latest_snapshot_revision(self, origin_url, | ||||
| previous_swh_revision=None): | previous_swh_revision=None): | ||||
| """Look for latest snapshot revision and returns it if any. | """Look for latest snapshot revision and returns it if any. | ||||
| Args: | Args: | ||||
| origin_url (str): Origin identifier | origin_url (str): Origin identifier | ||||
| previous_swh_revision: (optional) id of a possible | previous_swh_revision: (optional) id of a possible | ||||
| previous swh revision | previous swh revision | ||||
| ▲ Show 20 Lines • Show All 296 Lines • ▼ Show 20 Lines | def prepare(self, *args, **kwargs): | ||||
| if self.destination_path: | if self.destination_path: | ||||
| local_dirname = self.destination_path | local_dirname = self.destination_path | ||||
| else: | else: | ||||
| local_dirname = tempfile.mkdtemp( | local_dirname = tempfile.mkdtemp( | ||||
| suffix='-%s' % os.getpid(), | suffix='-%s' % os.getpid(), | ||||
| prefix=TEMPORARY_DIR_PREFIX_PATTERN, | prefix=TEMPORARY_DIR_PREFIX_PATTERN, | ||||
| dir=self.temp_directory) | dir=self.temp_directory) | ||||
| self.svnrepo = self.get_svn_repo( | self.svnrepo = svn.SvnRepo( | ||||
| self.svn_url, local_dirname, self.origin_url) | self.svn_url, local_dirname, self.origin_url, | ||||
| self.max_content_length) | |||||
| try: | try: | ||||
| revision_start, revision_end, revision_parents = self.start_from( | revision_start, revision_end, revision_parents = self.start_from( | ||||
| self.last_known_swh_revision, self.start_from_scratch) | self.last_known_swh_revision, self.start_from_scratch) | ||||
| self.swh_revision_gen = self.process_svn_revisions( | self.swh_revision_gen = self.process_svn_revisions( | ||||
| self.svnrepo, revision_start, revision_end, revision_parents) | self.svnrepo, revision_start, revision_end, revision_parents) | ||||
| except SvnLoaderUneventful as e: | except SvnLoaderUneventful as e: | ||||
| self.log.warning(e) | self.log.warning(e) | ||||
| if self.latest_snapshot and 'snapshot' in self.latest_snapshot: | if self.latest_snapshot and 'snapshot' in self.latest_snapshot: | ||||
| ▲ Show 20 Lines • Show All 290 Lines • Show Last 20 Lines | |||||