diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py index 2e3edd9..f225d1b 100644 --- a/swh/loader/svn/loader.py +++ b/swh/loader/svn/loader.py @@ -1,537 +1,537 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Loader in charge of injecting either new or existing svn mirrors to swh-storage. """ import abc import datetime from swh.core import utils, hashutil from swh.model import git from swh.model.git import GitType from swh.loader.core.loader import SWHLoader from . import svn, converters from .utils import hashtree class SvnLoaderEventful(ValueError): """A wrapper exception to transit the swh_revision onto which the loading failed. """ def __init__(self, e, swh_revision): super().__init__(e) self.swh_revision = swh_revision class SvnLoaderUneventful(ValueError): pass class SvnLoaderHistoryAltered(ValueError): pass class BaseSvnLoader(SWHLoader, metaclass=abc.ABCMeta): """Base Svn loader to load one svn repository according to specific policies (only swh one now). The main entry point of this is (no need to override it) - def load(self, origin_visit, last_known_swh_revision=None): pass Inherit this class and then override the following functions: - def build_swh_revision(self, rev, commit, dir_id, parents) This is in charge of converting an svn revision to a compliant swh revision - def process_repository(self) This is in charge of processing the actual svn repository and store the result to swh storage. """ CONFIG_BASE_FILENAME = 'loader/svn' ADDITIONAL_CONFIG = { 'check_revision': ('int', 1000), } def __init__(self): super().__init__(logging_class='swh.loader.svn.SvnLoader') self.check_revision = self.config['check_revision'] @abc.abstractmethod def swh_revision_hash_tree_at_svn_revision(self, revision): """Compute and return the hash tree at a given svn revision. Args: rev (int): the svn revision we want to check Returns: The hash tree directory as bytes. """ pass @abc.abstractmethod def get_svn_repo(self, svn_url, destination_path, origin): """Instantiates the needed svnrepo collaborator to permit reading svn repository. Args: svn_url: the svn repository url to read from destination_path: the local path on disk to compute data origin: the corresponding origin Returns: Instance of swh.loader.svn.svn clients """ raise NotImplementedError @abc.abstractmethod def build_swh_revision(self, rev, commit, dir_id, parents): """Convert an svn revision to an swh one according to the loader's policy (git-svn or swh). Args: rev: the svn revision number commit: dictionary with keys 'author_name', 'author_date', 'rev', 'message' dir_id: the hash tree computation parents: the revision's parents Returns: The swh revision """ raise NotImplementedError @abc.abstractmethod def process_repository(self, origin_visit, last_known_swh_revision=None): """The main idea of this function is to: - iterate over the svn commit logs - extract the svn commit log metadata - compute the hashes from the current directory down to the file - compute the equivalent swh revision - send all those objects for storage - create an swh occurrence pointing to the last swh revision seen - send that occurrence for storage in swh-storage. """ raise NotImplementedError def process_svn_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process revisions from revision_start to revision_end and send to swh for storage. At each svn revision, checkout the repository, compute the tree hash and blobs and send for swh storage to store. Then computes and yields the swh revision. Note that at every self.check_revision, an svn export is done and a hash tree is computed to check that no divergence occurred. Yields: swh revision as a dictionary with keys, sha1_git, sha1, etc... """ gen_revs = svnrepo.swh_hash_data_per_revision( revision_start, revision_end) swh_revision = None count = 0 for rev, nextrev, commit, objects_per_path in gen_revs: count += 1 # Send the associated contents/directories self.maybe_load_contents( git.objects_per_type(GitType.BLOB, objects_per_path)) self.maybe_load_directories( git.objects_per_type(GitType.TREE, objects_per_path)) # compute the fs tree's checksums dir_id = objects_per_path[b'']['checksums']['sha1_git'] swh_revision = self.build_swh_revision( rev, commit, dir_id, revision_parents[rev]) swh_revision['id'] = git.compute_revision_sha1_git( swh_revision) self.log.debug('rev: %s, swhrev: %s, dir: %s' % ( rev, hashutil.hash_to_hex(swh_revision['id']), hashutil.hash_to_hex(dir_id))) if (count % self.check_revision) == 0: # hash computation check self.log.info('Checking hash computations on revision %s...' % rev) checked_dir_id = self.swh_revision_hash_tree_at_svn_revision( rev) if checked_dir_id != dir_id: err = 'Hash tree computation divergence detected (%s != %s), stopping!' % ( # noqa hashutil.hash_to_hex(dir_id), hashutil.hash_to_hex(checked_dir_id) ) raise ValueError(err) if nextrev: revision_parents[nextrev] = [swh_revision['id']] yield swh_revision def process_swh_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process and store revision to swh (sent by blocks of 'revision_packet_size') Returns: The latest revision stored. """ try: swh_revision_gen = self.process_svn_revisions(svnrepo, revision_start, revision_end, revision_parents) revs = [] for revisions in utils.grouper( swh_revision_gen, self.config['revision_packet_size']): revs = list(revisions) self.log.info('Processed %s revisions: [%s, ...]' % ( len(revs), hashutil.hash_to_hex(revs[0]['id']))) self.maybe_load_revisions(revs) except Exception as e: if revs: # flush remaining revisions self.maybe_load_revisions(revs) # Take the last one as the last known revisions known_swh_rev = revs[-1] # Then notify something is wrong, and we stopped at that rev. raise SvnLoaderEventful(e, swh_revision={ 'id': known_swh_rev['id'], }) raise e return revs[-1] def process_swh_occurrence(self, revision, origin_visit): """Process and load the occurrence pointing to the latest revision. """ occ = converters.build_swh_occurrence(revision['id'], origin_visit['origin'], origin_visit['visit']) self.log.debug('occ: %s' % occ) self.maybe_load_occurrences([occ]) def close_success(self): self.close_fetch_history_success(self.fetch_history_id) self.storage.origin_visit_update(self.origin_visit['origin'], self.origin_visit['visit'], status='full') def close_failure(self): self.close_fetch_history_failure(self.fetch_history_id) self.storage.origin_visit_update(self.origin_visit['origin'], self.origin_visit['visit'], status='partial') def load(self, *args, **kwargs): """Load a svn repository in swh. Checkout the svn repository locally in destination_path. Args: - origin_visit: (mandatory) The current origin visit - last_known_swh_revision: (Optional) Hash id of known swh revision already visited in a previous visit Returns: Dictionary with the following keys: - eventful: (mandatory) is the loading being eventful or not - completion: (mandatory) 'full' if complete, 'partial' otherwise - state: (optional) if the completion was partial, this gives the state to pass along for the next schedule """ self.prepare(*args, **kwargs) try: latest_rev = self.process_repository( self.origin_visit, self.last_known_swh_revision) except SvnLoaderEventful as e: self.log.error('Eventful partial visit. Detail: %s' % e) latest_rev = e.swh_revision self.process_swh_occurrence(latest_rev, self.origin_visit) self.close_failure() return { 'eventful': True, 'completion': 'partial', 'state': { 'origin': self.origin_visit['origin'], 'revision': hashutil.hash_to_hex(latest_rev['id']) } } except (SvnLoaderHistoryAltered, SvnLoaderUneventful) as e: self.log.error('Uneventful visit. Detail: %s' % e) # FIXME: This fails because latest_rev is not bound # self.process_swh_occurrence(latest_rev, self.origin_visit) self.close_failure() return { 'eventful': False, } except Exception as e: self.close_failure() raise e else: self.process_swh_occurrence(latest_rev, self.origin_visit) self.close_success() return { 'eventful': True, 'completion': 'full', } finally: self.clean() @abc.abstractmethod def clean(self): """Clean up after working. """ pass def prepare(self, *args, **kwargs): """ Prepare origin, fetch_origin, origin_visit Then load a svn repository. Then close origin_visit, fetch_history according to status success or failure. First: - creates an origin if it does not exist - creates a fetch_history entry - creates an origin_visit - Then loads the svn repository """ destination_path = kwargs['destination_path'] # local svn url svn_url = kwargs['svn_url'] if 'origin' not in kwargs: # first time, we'll create the origin origin = { 'url': svn_url, 'type': 'svn', } origin['id'] = self.storage.origin_add_one(origin) else: origin = { 'id': kwargs['origin'], 'url': svn_url, 'type': 'svn' } if 'swh_revision' in kwargs: self.last_known_swh_revision = hashutil.hex_to_hash( kwargs['swh_revision']) else: self.last_known_swh_revision = None self.svnrepo = self.get_svn_repo(svn_url, destination_path, origin) self.origin_id = origin['id'] self.fetch_history_id = self.open_fetch_history() date_visit = datetime.datetime.now(tz=datetime.timezone.utc) self.origin_visit = self.storage.origin_visit_add( self.origin_id, date_visit) class SWHSvnLoader(BaseSvnLoader): """Swh svn loader is the main implementation destined for production. This implementation is able to deal with update on known svn repository. Default policy: Keep data as close as possible from the original svn data. We only add information that are needed for update or continuing from last known revision (svn revision and svn repository's uuid). """ def __init__(self): super().__init__() def clean(self): """Clean after oneself. This is in charge to flush the remaining data to write in swh storage. And to clean up the svn repository's working representation on disk. """ self.flush() self.svnrepo.clean_fs() def swh_revision_hash_tree_at_svn_revision(self, revision): """Compute a given hash tree at specific revision. """ - local_url = self.svnrepo.export_temporary(revision) + local_dirname, local_url = self.svnrepo.export_temporary(revision) h = hashtree(local_url)['sha1_git'] - self.svnrepo.clean_fs(local_url) + self.svnrepo.clean_fs(local_dirname) return h def get_svn_repo(self, svn_url, destination_path, origin): return svn.SWHSvnRepo( svn_url, origin['id'], self.storage, destination_path=destination_path) def swh_previous_revision(self, prev_swh_revision=None): """Retrieve swh's previous revision if any. """ return self.svnrepo.swh_previous_revision(prev_swh_revision) def check_history_not_altered(self, svnrepo, revision_start, swh_rev): """Given a svn repository, check if the history was not tampered with. """ revision_id = swh_rev['id'] parents = swh_rev['parents'] hash_data_per_revs = svnrepo.swh_hash_data_at_revision(revision_start) rev = revision_start rev, _, commit, objects_per_path = list(hash_data_per_revs)[0] dir_id = objects_per_path[b'']['checksums']['sha1_git'] swh_revision = self.build_swh_revision(rev, commit, dir_id, parents) swh_revision_id = git.compute_revision_sha1_git(swh_revision) return swh_revision_id == revision_id def build_swh_revision(self, rev, commit, dir_id, parents): """Build the swh revision dictionary. This adds: - the 'synthetic' flag to true - the 'extra_headers' containing the repository's uuid and the svn revision number. Args: rev: the svn revision commit: the commit metadata dir_id: the upper tree's hash identifier parents: the parents' identifiers Returns: The swh revision corresponding to the svn revision. """ return converters.build_swh_revision(rev, commit, self.svnrepo.uuid, dir_id, parents) def init_from(self, partial_swh_revision, previous_swh_revision): """Function to determine from where to start from. Args: - partial_swh_revision: A known revision from which the previous loading did not finish. - known_previous_revision: A known revision from which the previous loading did finish. Returns: The revision from which to start or None if nothing (fresh start). """ if partial_swh_revision and not previous_swh_revision: return partial_swh_revision if not partial_swh_revision and previous_swh_revision: return previous_swh_revision if partial_swh_revision and previous_swh_revision: # will determine from which to start from extra_headers1 = dict( partial_swh_revision['metadata']['extra_headers']) extra_headers2 = dict( previous_swh_revision['metadata']['extra_headers']) rev_start1 = int(extra_headers1['svn_revision']) rev_start2 = int(extra_headers2['svn_revision']) if rev_start1 <= rev_start2: return previous_swh_revision return partial_swh_revision return None def process_repository(self, origin_visit, last_known_swh_revision=None): svnrepo = self.svnrepo # default configuration revision_start = 1 revision_parents = { revision_start: [] } # Check if we already know a previous revision for that origin swh_rev = self.swh_previous_revision() # Determine from which known revision to start swh_rev = self.init_from(last_known_swh_revision, previous_swh_revision=swh_rev) if swh_rev: # Yes, we do. Try and update it. extra_headers = dict(swh_rev['metadata']['extra_headers']) revision_start = int(extra_headers['svn_revision']) revision_parents = { revision_start: swh_rev['parents'], } self.log.debug('svn export --ignore-keywords %s@%s' % ( svnrepo.remote_url, revision_start)) if swh_rev and not self.check_history_not_altered( svnrepo, revision_start, swh_rev): msg = 'History of svn %s@%s history modified. Skipping...' % ( # noqa svnrepo.remote_url, revision_start) self.log.warn(msg) raise SvnLoaderHistoryAltered else: # now we know history is ok, we start at next revision revision_start = revision_start + 1 # and the parent become the latest know revision for # that repository revision_parents[revision_start] = [swh_rev['id']] revision_end = svnrepo.head_revision() self.log.info('[revision_start-revision_end]: [%s-%s]' % ( revision_start, revision_end)) if revision_start > revision_end and revision_start is not 1: self.log.info('%s@%s already injected.' % ( svnrepo.remote_url, revision_end)) raise SvnLoaderUneventful self.log.info('Processing %s.' % svnrepo) # process and store revision to swh (sent by by blocks of # 'revision_packet_size') return self.process_swh_revisions( svnrepo, revision_start, revision_end, revision_parents) diff --git a/swh/loader/svn/svn.py b/swh/loader/svn/svn.py index c91c5ee..558024c 100644 --- a/swh/loader/svn/svn.py +++ b/swh/loader/svn/svn.py @@ -1,369 +1,370 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """SVN client in charge of iterating over svn logs and yield commit representations including the hash tree/content computations per svn commit. """ import os import tempfile import shutil from subvertpy.ra import RemoteAccess, Auth, get_username_provider from subvertpy import client, properties from swh.model import git from . import ra, utils, converters # When log message contains empty data DEFAULT_AUTHOR_MESSAGE = '' class SvnRepoException(ValueError): def __init__(self, svnrepo, e): super().__init__(e) self.svnrepo = svnrepo class BaseSvnRepo(): """Base svn repository representation for swh. To override some of the behavior regarding the message log properties, you can instantiate a subclass of this class and override: - def convert_commit_author(self, author) - def convert_commit_message(self, msg) - def convert_commit_date(self, date) cf. GitSvnSvnRepo, SwhSvnRepo for instanciation example. """ def __init__(self, remote_url, origin_id, storage, destination_path=None): self.remote_url = remote_url.rstrip('/') self.storage = storage self.origin_id = origin_id if destination_path: os.makedirs(destination_path, exist_ok=True) self.root_dir = destination_path else: self.root_dir = '/tmp' auth = Auth([get_username_provider()]) # one connection for log iteration self.conn_log = RemoteAccess(self.remote_url, auth=auth) # another for replay self.conn = RemoteAccess(self.remote_url, auth=auth) # one client for update operation self.client = client.Client(auth=auth) self.local_dirname = tempfile.mkdtemp( suffix='.swh.loader', prefix='tmp.', dir=self.root_dir) local_name = os.path.basename(self.remote_url) self.local_url = os.path.join(self.local_dirname, local_name).encode( 'utf-8') self.uuid = self.conn.get_uuid().encode('utf-8') def __str__(self): return str({ 'remote_url': self.remote_url, 'local_url': self.local_url, 'uuid': self.uuid, 'swh-origin': self.origin_id }) def head_revision(self): """Retrieve current revision of the repository's working copy. """ return self.conn.get_latest_revnum() def initial_revision(self): """Retrieve the initial revision from which the remote url appeared. Note: This should always be 1 since we won't be dealing with in-depth url. """ return 1 def convert_commit_message(self, msg): """Do something with message (e.g add extra line, etc...) cf. SvnRepo for a simple implementation. Args: msg (str): the commit message to convert. Returns: The transformed message as bytes. """ raise NotImplementedError('Should be overridden by subclass.') def convert_commit_date(self, date): """Convert the message date (e.g, convert into timestamp or whatever makes sense to you.). Args: date (str): the commit date to convert. Returns: The transformed date. """ raise NotImplementedError('Should be overridden by subclass.') def convert_commit_author(self, author): """Convert the commit author (e.g, convert into dict or whatever makes sense to you.). Args: author (str): the commit author to convert. Returns: The transformed author as dict. """ raise NotImplementedError('Should be overridden by subclass.') def __to_entry(self, log_entry): changed_paths, rev, revprops, has_children = log_entry author_date = self.convert_commit_date( revprops.get(properties.PROP_REVISION_DATE)) author = self.convert_commit_author( revprops.get(properties.PROP_REVISION_AUTHOR)) message = self.convert_commit_message( revprops.get(properties.PROP_REVISION_LOG, DEFAULT_AUTHOR_MESSAGE)) return { 'rev': rev, 'author_date': author_date, 'author_name': author, 'message': message, } def logs(self, revision_start, revision_end): """Stream svn logs between revision_start and revision_end by chunks of block_size logs. Yields revision and associated revision information between the revision start and revision_end. Args: revision_start: the svn revision starting bound revision_end: the svn revision ending bound Yields: tuple of revisions and logs. revisions: list of revisions in order logs: Dictionary with key revision number and value the log entry. The log entry is a dictionary with the following keys: - author_date: date of the commit - author_name: name of the author - message: commit message """ for log_entry in self.conn_log.iter_log(paths=None, start=revision_start, end=revision_end, discover_changed_paths=False): yield self.__to_entry(log_entry) def export(self, revision): """Export the repository to a given version. """ self.client.export(self.remote_url, to=self.local_url.decode('utf-8'), rev=revision, ignore_keywords=True) def export_temporary(self, revision): """Export the repository to a given revision in a temporary location. This is up to the caller of this function to clean up the temporary location when done (cf. self.clean_fs method) Args: revision: Revision to export at Returns: - The local_url where the repository was exported. + The tuple local_dirname the temporary location root + folder, local_url where the repository was exported. """ local_dirname = tempfile.mkdtemp( prefix='check-revision-%s.' % revision, dir=self.root_dir) local_name = os.path.basename(self.remote_url) local_url = os.path.join(local_dirname, local_name) self.client.export( self.remote_url, to=local_url, rev=revision, ignore_keywords=True) - return local_url + return local_dirname, local_url def swh_previous_revision(self, previous_swh_revision=None): """Look for possible existing revision in swh. Args: previous_swh_revision: (optional) id of a possible previous swh revision Returns: If previous_swh_revision is not None and do exists, returns the complete instance. Otherwise, check for a possible occurrence and returns the targeted complete revision if it does exists. Otherwise, returns None. """ storage = self.storage # got no previous revision, will check if some occurrence # already exists for that origin if not previous_swh_revision: occ = storage.occurrence_get(self.origin_id) if occ: revision_id = occ[0]['target'] revisions = storage.revision_get([revision_id]) if revisions: return revisions[0] else: revs = storage.revision_get([previous_swh_revision]) if revs: return revs[0] def swh_hash_data_per_revision(self, start_revision, end_revision): """Compute swh hash data per each revision between start_revision and end_revision. Args: start_revision: starting revision end_revision: ending revision Yields: tuple (rev, nextrev, commit, objects_per_path) - rev: current revision - nextrev: next revision - commit: commit data (author, date, message) for such revision - objects_per_path: dictionary of path, swh hash data with type """ hashes = {} for commit in self.logs(start_revision, end_revision): rev = commit['rev'] hashes = self.swhreplay.compute_hashes(rev) if rev == end_revision: nextrev = None else: nextrev = rev + 1 yield rev, nextrev, commit, hashes def swh_hash_data_at_revision(self, revision): """Compute the hash data at revision. Expected to be used for update only. """ # Update the disk at revision self.export(revision) # Compute the current hashes on disk hashes = git.compute_hashes_from_directory(self.local_url) hashes = utils.convert_hashes_with_relative_path( hashes, rootpath=self.local_url) # Update the replay collaborator with the right state self.swhreplay = ra.SWHReplay( conn=self.conn, rootpath=self.local_url, objects=hashes) # Retrieve the commit information for revision commit = list(self.logs(revision, revision))[0] yield revision, revision + 1, commit, hashes def clean_fs(self, local_dirname=None): """Clean up the local working copy. Args: local_dirname (str): Path to remove recursively if provided. Otherwise, remove the temporary upper root tree used for svn repository loading. """ if local_dirname: shutil.rmtree(local_dirname) else: shutil.rmtree(self.local_dirname) class SWHSvnRepo(BaseSvnRepo): """This class does exactly as BaseSvnRepo except for: - the commit message which is simply encoded - the commit author is left as is. - the commit timestamp is left as is. """ def __init__(self, remote_url, origin_id, storage, destination_path=None): super().__init__(remote_url, origin_id, storage, destination_path=destination_path) self.swhreplay = ra.SWHReplay( conn=self.conn, rootpath=self.local_url) def convert_commit_message(self, msg): """Simply encode the commit message. Args: msg (str): the commit message to convert. Returns: The transformed message as bytes. """ return msg.encode('utf-8') def convert_commit_date(self, date): """Convert the message commit date into a timestamp in swh format. The precision is kept. Args: date (str): the commit date to convert. Returns: The transformed date. """ return converters.svn_date_to_swh_date(date) def convert_commit_author(self, author): """Convert the commit author into an swh person. The user becomes a dictionary of the form: { name: author, email: '', fullname: author } Args: author (str): the commit author to convert. Returns: The transformed author as dict. """ return converters.svn_author_to_swh_person(author)