diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py index a069575..2f34b14 100644 --- a/swh/loader/svn/loader.py +++ b/swh/loader/svn/loader.py @@ -1,544 +1,548 @@ # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Loader in charge of injecting either new or existing svn mirrors to swh-storage. """ import os import shutil from swh.core import utils from swh.model import hashutil from swh.model.from_disk import Directory from swh.model.identifiers import identifier_to_bytes, revision_identifier from swh.model.identifiers import snapshot_identifier from swh.loader.core.loader import SWHLoader from . import svn, converters from .utils import init_svn_repo_from_archive_dump DEFAULT_BRANCH = b'master' def _revision_id(revision): return identifier_to_bytes(revision_identifier(revision)) def build_swh_snapshot(revision_id, branch=DEFAULT_BRANCH): """Build a swh snapshot from the revision id, origin id, and visit. """ return { 'id': None, 'branches': { branch: { 'target': revision_id, 'target_type': 'revision', } } } class SvnLoaderEventful(ValueError): """A wrapper exception to transit the swh_revision onto which the loading failed. """ def __init__(self, e, swh_revision): super().__init__(e) self.swh_revision = swh_revision class SvnLoaderUneventful(ValueError): """'Loading did nothing' exception to transit the latest known snapshot. """ def __init__(self, e): super().__init__(e) class SvnLoaderHistoryAltered(ValueError): def __init__(self, e): super().__init__(e) class SWHSvnLoader(SWHLoader): """Swh svn loader to load an svn repository The repository is either remote or local. The loader deals with update on an already previously loaded repository. Default policy: Keep data as close as possible from the original svn data. We only add information that are needed for update or continuing from last known revision (svn revision and svn repository's uuid). """ CONFIG_BASE_FILENAME = 'loader/svn' ADDITIONAL_CONFIG = { 'check_revision': ('int', 1000), } def __init__(self): super().__init__(logging_class='swh.loader.svn.SvnLoader') self.check_revision = self.config['check_revision'] self.origin_id = None def cleanup(self): """Clean up the svn repository's working representation on disk. """ self.svnrepo.clean_fs() def swh_revision_hash_tree_at_svn_revision(self, revision): """Compute and return the hash tree at a given svn revision. Args: rev (int): the svn revision we want to check Returns: The hash tree directory as bytes. """ local_dirname, local_url = self.svnrepo.export_temporary(revision) h = Directory.from_disk(path=local_url).hash self.svnrepo.clean_fs(local_dirname) return h def get_svn_repo(self, svn_url, destination_path, origin): """Instantiates the needed svnrepo collaborator to permit reading svn repository. Args: svn_url (str): the svn repository url to read from destination_path (str): the local path on disk to compute data origin (int): the corresponding origin Returns: Instance of :mod:`swh.loader.svn.svn` clients """ return svn.SWHSvnRepo( svn_url, origin['id'], self.storage, destination_path=destination_path) def swh_latest_snapshot_revision(self, origin_id, previous_swh_revision=None): """Look for latest snapshot revision and returns it if any. Args: origin_id (int): Origin identifier previous_swh_revision: (optional) id of a possible previous swh revision Returns: dict: The latest known point in time. Dict with keys: 'revision': latest visited revision 'snapshot': latest snapshot If None is found, return an empty dict. """ storage = self.storage if not previous_swh_revision: # check latest snapshot's revision latest_snap = storage.snapshot_get_latest(origin_id) if latest_snap: branches = latest_snap.get('branches') if not branches: return {} branch = branches.get(DEFAULT_BRANCH) if not branch: return {} target_type = branch['target_type'] if target_type != 'revision': return {} previous_swh_revision = branch['target'] else: return {} revs = list(storage.revision_get([previous_swh_revision])) if revs: return { 'snapshot': latest_snap, 'revision': revs[0] } return {} def build_swh_revision(self, rev, commit, dir_id, parents): """Build the swh revision dictionary. This adds: - the `'synthetic`' flag to true - the '`extra_headers`' containing the repository's uuid and the svn revision number. Args: rev (dict): the svn revision commit (dict): the commit metadata dir_id (bytes): the upper tree's hash identifier parents ([bytes]): the parents' identifiers Returns: The swh revision corresponding to the svn revision. """ return converters.build_swh_revision(rev, commit, self.svnrepo.uuid, dir_id, parents) def check_history_not_altered(self, svnrepo, revision_start, swh_rev): """Given a svn repository, check if the history was not tampered with. """ revision_id = swh_rev['id'] parents = swh_rev['parents'] hash_data_per_revs = svnrepo.swh_hash_data_at_revision(revision_start) rev = revision_start rev, _, commit, _, root_dir = list(hash_data_per_revs)[0] dir_id = root_dir.hash swh_revision = self.build_swh_revision(rev, commit, dir_id, parents) swh_revision_id = _revision_id(swh_revision) return swh_revision_id == revision_id def process_repository(self, origin_visit, last_known_swh_revision=None, start_from_scratch=False): """The main idea of this function is to: - iterate over the svn commit logs - extract the svn commit log metadata - compute the hashes from the current directory down to the file - compute the equivalent swh revision - send all those objects for storage - create an swh occurrence pointing to the last swh revision seen - send that occurrence for storage in swh-storage. """ svnrepo = self.svnrepo - # default configuration - revision_start = 1 + revision_head = svnrepo.head_revision() + if revision_head == 0: # empty repository case + revision_start = 0 + revision_end = 0 + else: # default configuration + revision_start = svnrepo.initial_revision() + revision_end = revision_head + revision_parents = { revision_start: [] } if not start_from_scratch: # Check if we already know a previous revision for that origin if self.latest_snapshot: swh_rev = self.latest_snapshot['revision'] else: swh_rev = None # Determine from which known revision to start swh_rev = self.init_from(last_known_swh_revision, previous_swh_revision=swh_rev) if swh_rev: # Yes, we know a previous revision. Try and update it. extra_headers = dict(swh_rev['metadata']['extra_headers']) revision_start = int(extra_headers['svn_revision']) revision_parents = { revision_start: swh_rev['parents'], } self.log.debug('svn export --ignore-keywords %s@%s' % ( svnrepo.remote_url, revision_start)) if swh_rev and not self.check_history_not_altered( svnrepo, revision_start, swh_rev): msg = 'History of svn %s@%s altered. ' \ 'Skipping...' % ( svnrepo.remote_url, revision_start) raise SvnLoaderHistoryAltered(msg) # now we know history is ok, we start at next revision revision_start = revision_start + 1 # and the parent become the latest know revision for # that repository revision_parents[revision_start] = [swh_rev['id']] - revision_end = svnrepo.head_revision() - self.log.info('[revision_start-revision_end]: [%s-%s]' % ( revision_start, revision_end)) if revision_start > revision_end and revision_start is not 1: msg = '%s@%s already injected.' % (svnrepo.remote_url, revision_end) raise SvnLoaderUneventful(msg) self.log.info('Processing %s.' % svnrepo) # process and store revision to swh (sent by by blocks of # 'revision_packet_size') return self.process_swh_revisions( svnrepo, revision_start, revision_end, revision_parents) def process_svn_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process revisions from revision_start to revision_end and send to swh for storage. At each svn revision, checkout the repository, compute the tree hash and blobs and send for swh storage to store. Then computes and yields the swh revision. Note that at every self.check_revision, an svn export is done and a hash tree is computed to check that no divergence occurred. Yields: swh revision as a dictionary with keys, sha1_git, sha1, etc... """ gen_revs = svnrepo.swh_hash_data_per_revision( revision_start, revision_end) swh_revision = None count = 0 for rev, nextrev, commit, new_objects, root_directory in gen_revs: count += 1 # Send the associated contents/directories self.maybe_load_contents(new_objects.get('content', {}).values()) self.maybe_load_directories( new_objects.get('directory', {}).values()) # compute the fs tree's checksums dir_id = root_directory.hash swh_revision = self.build_swh_revision( rev, commit, dir_id, revision_parents[rev]) swh_revision['id'] = _revision_id(swh_revision) self.log.debug('rev: %s, swhrev: %s, dir: %s' % ( rev, hashutil.hash_to_hex(swh_revision['id']), hashutil.hash_to_hex(dir_id))) if (count % self.check_revision) == 0: # hash computation check self.log.info('Checking hash computations on revision %s...' % rev) checked_dir_id = self.swh_revision_hash_tree_at_svn_revision( rev) if checked_dir_id != dir_id: err = 'Hash tree computation divergence detected ' \ '(%s != %s), stopping!' % ( hashutil.hash_to_hex(dir_id), hashutil.hash_to_hex(checked_dir_id)) raise ValueError(err) if nextrev: revision_parents[nextrev] = [swh_revision['id']] yield swh_revision def process_swh_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process and store revision to swh (sent by blocks of revision_packet_size) Returns: The latest revision stored. """ try: swh_revision_gen = self.process_svn_revisions(svnrepo, revision_start, revision_end, revision_parents) revs = [] for revisions in utils.grouper( swh_revision_gen, self.config['revision_packet_size']): revs = list(revisions) self.log.info('Processed %s revisions: [%s, ...]' % ( len(revs), hashutil.hash_to_hex(revs[0]['id']))) self.maybe_load_revisions(revs) except Exception as e: if revs: # flush remaining revisions self.maybe_load_revisions(revs) # Take the last one as the last known revisions known_swh_rev = revs[-1] _id = known_swh_rev.get('id') if not _id: _id = _revision_id(known_swh_rev) # Then notify something is wrong, and we stopped at that rev. raise SvnLoaderEventful(e, swh_revision={ 'id': _id, }) raise e return revs[-1] def process_swh_snapshot(self, revision=None, snapshot=None): """Create the snapshot either from existing snapshot or revision. """ if snapshot: snap = snapshot elif revision: snap = build_swh_snapshot(revision['id']) snap['id'] = identifier_to_bytes(snapshot_identifier(snap)) else: return None self.log.debug('snapshot: %s' % snap) self.maybe_load_snapshot(snap) def prepare(self, *args, **kwargs): self.args = args destination_path = kwargs['destination_path'] # local svn url svn_url = kwargs['svn_url'] origin_url = kwargs.get('origin_url') self.visit_date = kwargs.get('visit_date') self.start_from_scratch = kwargs.get('start_from_scratch', False) origin = { 'url': origin_url if origin_url else svn_url, 'type': 'svn', } self.origin_id = self.send_origin(origin) origin['id'] = self.origin_id self.origin = origin if 'swh_revision' in kwargs: self.last_known_swh_revision = hashutil.hash_to_bytes( kwargs['swh_revision']) else: self.last_known_swh_revision = None self.latest_snapshot = self.swh_latest_snapshot_revision( self.origin_id, self.last_known_swh_revision) self.svnrepo = self.get_svn_repo(svn_url, destination_path, origin) def get_origin(self): """Retrieve the origin we are working with (setup-ed in the prepare method) """ return self.origin # set in prepare method def fetch_data(self): """We need to fetch and stream the data to store directly. So fetch_data do actually nothing. The method ``store_data`` below is in charge to do everything, fetch and store. """ pass def store_data(self): """We need to fetch and stream the data to store directly because there is too much data and state changes. Everything is intertwined together (We receive patch and apply on disk and compute at the hashes at the same time) So every data to fetch and store is done here. Note: origin_visit and last_known_swh_revision must have been set in the prepare method. """ origin_visit = {'origin': self.origin_id, 'visit': self.visit} try: latest_rev = self.process_repository( origin_visit, last_known_swh_revision=self.last_known_swh_revision, start_from_scratch=self.start_from_scratch) except SvnLoaderUneventful as e: # uneventful visit self.log.info('Uneventful visit. Detail: %s' % e) if self.latest_snapshot and 'snapshot' in self.latest_snapshot: snapshot = self.latest_snapshot['snapshot'] self.process_swh_snapshot(snapshot=snapshot) except SvnLoaderEventful as e: self.log.error('Eventful partial visit. Detail: %s' % e) latest_rev = e.swh_revision self.process_swh_snapshot(revision=latest_rev) raise except Exception as e: if self.latest_snapshot and 'snapshot' in self.latest_snapshot: snapshot = self.latest_snapshot['snapshot'] self.process_swh_snapshot(snapshot=snapshot) raise else: self.process_swh_snapshot(revision=latest_rev) def init_from(self, partial_swh_revision, previous_swh_revision): """Function to determine from where to start from. Args: partial_swh_revision: A known revision from which the previous loading did not finish. known_previous_revision: A known revision from which the previous loading did finish. Returns: The revision from which to start or None if nothing (fresh start). """ if partial_swh_revision and not previous_swh_revision: return partial_swh_revision if not partial_swh_revision and previous_swh_revision: return previous_swh_revision if partial_swh_revision and previous_swh_revision: # will determine from which to start from extra_headers1 = dict( partial_swh_revision['metadata']['extra_headers']) extra_headers2 = dict( previous_swh_revision['metadata']['extra_headers']) rev_start1 = int(extra_headers1['svn_revision']) rev_start2 = int(extra_headers2['svn_revision']) if rev_start1 <= rev_start2: return previous_swh_revision return partial_swh_revision return None class SWHSvnLoaderFromDumpArchive(SWHSvnLoader): """Uncompress an archive containing an svn dump, mount the svn dump as an svn repository and load said repository. """ def __init__(self, archive_path): super().__init__() self.log.info('Archive to mount and load %s' % archive_path) self.temp_dir, self.repo_path = init_svn_repo_from_archive_dump( archive_path) def cleanup(self): super().cleanup() if self.temp_dir and os.path.exists(self.temp_dir): self.log.debug('Clean up temp directory %s for project %s' % ( self.temp_dir, os.path.basename(self.repo_path))) shutil.rmtree(self.temp_dir)