diff --git a/swh/loader/svn/converters.py b/swh/loader/svn/converters.py index bc43a2d..72e2f35 100644 --- a/swh/loader/svn/converters.py +++ b/swh/loader/svn/converters.py @@ -1,238 +1,238 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from email import utils from swh.core import hashutil from .utils import strdate_to_timestamp def svn_date_to_gitsvn_date(strdate): """Convert a string date to an swh one. Args: strdate: A string formatted for .utils.strdate_to_timestamp to do its jobs Returns: An swh date format with an integer timestamp. """ return { 'timestamp': int(strdate_to_timestamp(strdate)), 'offset': 0 } def svn_date_to_swh_date(strdate): """Convert a string date to an swh one. Args: strdate: A string formatted for .utils.strdate_to_timestamp to do its jobs Returns: An swh date format """ return { 'timestamp': strdate_to_timestamp(strdate), 'offset': 0 } def svn_author_to_swh_person(author): """Convert an svn author to an swh person. Default policy: No information is added. Args: author (string): the svn author (in bytes) Returns: a dictionary with keys: fullname: the author's associated fullname name: the author's associated name email: None (no email in svn) """ if not author: return {'fullname': b'', 'name': None, 'email': None} author = author.encode('utf-8') if b'<' in author and b'>' in author: name, email = utils.parseaddr(author.decode('utf-8')) return { 'fullname': author, 'name': name.encode('utf-8'), 'email': email.encode('utf-8') } return {'fullname': author, 'email': None, 'name': author} def svn_author_to_gitsvn_person(author, repo_uuid): """Convert an svn author to a person suitable for insertion. Default policy: If no email is found, the email is created using the author and the repo_uuid. Args: author (string): the svn author (in bytes) repo_uuid (bytes): the repository's uuid Returns: a dictionary with keys: fullname: the author's associated fullname name: the author's associated name email: None (no email in svn) """ if not author: author = '(no author)' author = author.encode('utf-8') if b'<' in author and b'>' in author: name, email = utils.parseaddr(author.decode('utf-8')) return { 'fullname': author, 'name': name.encode('utf-8'), 'email': email.encode('utf-8') } # we'll construct the author's fullname the same way git svn does # 'user ' email = b'@'.join([author, repo_uuid]) return { 'fullname': b''.join([author, b' ', b'<', email, b'>']), 'name': author, 'email': email, } def build_swh_revision(rev, commit, repo_uuid, dir_id, parents): """Given a svn revision, build a swh revision. This adds an ['metadata']['extra-headers'] entry with the repository's uuid and the svn revision. Args: - rev: the svn revision number - commit: the commit metadata - repo_uuid: The repository's uuid - dir_id: the tree's hash identifier - parents: the revision's parents identifier Returns: The swh revision dictionary. """ author = commit['author_name'] msg = commit['message'] date = commit['author_date'] metadata = { 'extra_headers': [ ['svn_repo_uuid', repo_uuid], ['svn_revision', str(rev).encode('utf-8')] ] } return { 'date': date, 'committer_date': date, 'type': 'svn', 'directory': dir_id, 'message': msg, 'author': author, 'committer': author, 'synthetic': True, 'metadata': metadata, 'parents': parents, } def build_gitsvn_swh_revision(rev, commit, dir_id, parents): """Given a svn revision, build a swh revision. Args: - rev: the svn revision number - commit: the commit metadata - dir_id: the tree's hash identifier - parents: the revision's parents identifier Returns: The swh revision dictionary. """ author = commit['author_name'] msg = commit['message'] date = commit['author_date'] return { 'date': date, 'committer_date': date, 'type': 'svn', 'directory': dir_id, 'message': msg, 'author': author, 'committer': author, 'synthetic': True, 'metadata': None, 'parents': parents, } -def build_swh_occurrence(revision_id, origin_id, date): +def build_swh_occurrence(revision_id, origin_id, visit): """Build a swh occurrence from the revision id, origin id, and date. """ return {'branch': 'master', 'target': revision_id, 'target_type': 'revision', 'origin': origin_id, - 'date': date} + 'visit': visit} def loader_to_scheduler_revision(swh_revision): """To avoid serialization or scheduler storage problem, transform adequately the revision. FIXME: Should be more generically dealt with in swh-scheduler's side. The advantage to having it here is that we known what we store. """ if not swh_revision: return None metadata = swh_revision['metadata'] for entry in (e for e in metadata['extra_headers'] if isinstance(e[1], bytes)): entry[1] = entry[1].decode('utf-8') return { 'id': hashutil.hash_to_hex(swh_revision['id']), 'parents': [hashutil.hash_to_hex(parent) for parent in swh_revision['parents']], 'metadata': metadata } def scheduler_to_loader_revision(swh_revision): """If the known state (a revision) is already passed, it will be serializable ready but not loader ready. FIXME: Should be more generically dealt with in swh-scheduler's side. The advantage to having it here is that we known what we store. """ if not swh_revision: return None return { 'id': hashutil.hex_to_hash(swh_revision['id']), 'parents': [hashutil.hex_to_hash(parent) for parent in swh_revision['parents']], 'metadata': swh_revision['metadata'] } diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py index b5ff417..0b79546 100644 --- a/swh/loader/svn/loader.py +++ b/swh/loader/svn/loader.py @@ -1,468 +1,516 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Loader in charge of injecting either new or existing svn mirrors to swh-storage. """ -import datetime from swh.core import utils from swh.model import git, hashutil from swh.model.git import GitType from swh.loader.core.loader import SWHLoader from . import svn, converters class SvnLoaderException(ValueError): """A wrapper exception to transit the swh_revision onto which the loading failed. """ def __init__(self, e, swh_revision): super().__init__(e) self.swh_revision = swh_revision class BaseSvnLoader(SWHLoader): """Base Svn loader to load one svn repository. There exists 2 different policies: - git-svn one (not for production): cf. GitSvnSvnLoader - SWH one: cf. SWHSvnLoader The main entry point of this is (no need to override it) - - def load(self): + - def load(self, origin_visit, last_known_swh_revision=None): pass Inherit this class and then override the following functions: - def build_swh_revision(self, rev, commit, dir_id, parents) This is in charge of converting an svn revision to a compliant swh revision - def process_repository(self) This is in charge of processing the actual svn repository and store the result to swh storage. """ CONFIG_BASE_FILENAME = 'loader/svn.ini' def __init__(self, svn_url, destination_path, origin): super().__init__(origin['id'], logging_class='swh.loader.svn.SvnLoader') self.origin = origin def build_swh_revision(self, rev, commit, dir_id, parents): """Convert an svn revision to an swh one according to the loader's policy (git-svn or swh). Args: rev: the svn revision number commit: dictionary with keys 'author_name', 'author_date', 'rev', 'message' dir_id: the hash tree computation parents: the revision's parents Returns: The swh revision """ raise NotImplementedError('This should be overriden by subclass') - def process_repository(self): + def process_repository(self, origin_visit, last_known_swh_revision=None): """The main idea of this function is to: - iterate over the svn commit logs - extract the svn commit log metadata - compute the hashes from the current directory down to the file - compute the equivalent swh revision - send all those objects for storage - create an swh occurrence pointing to the last swh revision seen - send that occurrence for storage in swh-storage. """ raise NotImplementedError('This should be implemented in subclass.') def process_svn_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process revisions from revision_start to revision_end and send to swh for storage. At each svn revision, checkout the repository, compute the tree hash and blobs and send for swh storage to store. Then computes and yields the swh revision. Yields: swh revision """ gen_revs = svnrepo.swh_hash_data_per_revision( revision_start, revision_end) swh_revision = None for rev, nextrev, commit, objects_per_path in gen_revs: # Send the associated contents/directories self.maybe_load_contents( git.objects_per_type(GitType.BLOB, objects_per_path)) self.maybe_load_directories( git.objects_per_type(GitType.TREE, objects_per_path)) # compute the fs tree's checksums dir_id = objects_per_path[b'']['checksums']['sha1_git'] swh_revision = self.build_swh_revision( rev, commit, dir_id, revision_parents[rev]) swh_revision['id'] = git.compute_revision_sha1_git( swh_revision) self.log.debug('rev: %s, swhrev: %s, dir: %s' % ( rev, hashutil.hash_to_hex(swh_revision['id']), hashutil.hash_to_hex(dir_id))) if nextrev: revision_parents[nextrev] = [swh_revision['id']] yield swh_revision def process_swh_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process and store revision to swh (sent by blocks of 'revision_packet_size') Returns: The latest revision stored. """ try: swh_revision_gen = self.process_svn_revisions(svnrepo, revision_start, revision_end, revision_parents) revs = [] for revisions in utils.grouper( swh_revision_gen, self.config['revision_packet_size']): revs = list(revisions) self.log.info('Processed %s revisions: [%s, ...]' % ( len(revs), hashutil.hash_to_hex(revs[0]['id']))) self.maybe_load_revisions(revs) except Exception as e: if revs: # flush remaining revisions self.maybe_load_revisions(revs) # Take the last one as the last known revisions known_swh_rev = revs[-1] # Then notify something is wrong, and we stopped at that rev. raise SvnLoaderException(e, swh_revision={ 'id': known_swh_rev['id'], - 'parents': known_swh_rev['parents'], - 'metadata': known_swh_rev.get('metadata') }) else: raise SvnLoaderException(e, swh_revision=None) return revs[-1] - def process_swh_occurrence(self, revision, origin): + def process_swh_occurrence(self, revision, origin_visit): """Process and load the occurrence pointing to the latest revision. """ occ = converters.build_swh_occurrence(revision['id'], - origin['id'], - datetime.datetime.utcnow()) + origin_visit['origin'], + origin_visit['visit']) self.log.debug('occ: %s' % occ) self.maybe_load_occurrences([occ]) - def load(self, known_state=None): + def process_swh_origin_visit(self, origin_visit, status): + """Update origin_visit with status. + + """ + self.storage.origin_visit_update(origin_visit['origin'], + origin_visit['visit'], + status=status) + + def load(self, origin_visit, last_known_swh_revision=None): """Load a svn repository in swh. Checkout the svn repository locally in destination_path. Args: - - svn_url: svn repository url to import - - origin: Dictionary origin - - id: origin's id - - url: url origin we fetched - - type: type of the origin + - origin_visit: (mandatory) The current origin visit + - last_known_swh_revision: (Optional) Hash id of known swh revision + already visited in a previous visit Returns: Dictionary with the following keys: - - status: mandatory, the status result as a boolean - - stderr: optional when status is True, mandatory otherwise + - eventful: (mandatory) is the loading being eventful or not + - completion: (mandatory) 'full' if complete, 'partial' otherwise + - state: (optional) if the completion was partial, this + gives the state to pass along for the next schedule """ try: - self.process_repository(known_state) + return self.process_repository(origin_visit, + last_known_swh_revision) finally: - # flush eventual remaining data self.flush() self.svnrepo.clean_fs() - return {'status': True} - class GitSvnSvnLoader(BaseSvnLoader): """Git-svn like loader (compute hashes a-la git-svn) Notes: This implementation is: - NOT for production - NOT able to deal with update. Default policy: Its default policy is to enrich (or even alter) information at each svn revision. It will: - truncate the timestamp of the svn commit date - alter the user to be an email using the repository's uuid as mailserver (user -> user@) - fills in the gap for empty author with '(no author)' name - remove empty folder (thus not counting them during hash computation) The equivalent git command is: `git svn clone -q --no-metadata` """ def __init__(self, svn_url, destination_path, origin, svn_uuid=None): super().__init__(svn_url, destination_path, origin) # We don't want to persist result in git-svn policy self.config['send_contents'] = False self.config['send_directories'] = False self.config['send_revisions'] = False self.config['send_releases'] = False self.config['send_occurrences'] = False self.svnrepo = svn.GitSvnSvnRepo( svn_url, origin['id'], self.storage, destination_path=destination_path, svn_uuid=svn_uuid) def build_swh_revision(self, rev, commit, dir_id, parents): """Build the swh revision a-la git-svn. Args: rev: the svn revision commit: the commit metadata dir_id: the upper tree's hash identifier parents: the parents' identifiers Returns: The swh revision corresponding to the svn revision without any extra headers. """ return converters.build_gitsvn_swh_revision(rev, commit, dir_id, parents) - def process_repository(self, known_state=None): + def process_repository(self, origin_visit, last_known_swh_revision=None): """Load the repository's svn commits and process them as swh hashes. This does not: - deal with update - nor with the potential known state. """ - origin = self.origin svnrepo = self.svnrepo + # default configuration revision_start = 1 revision_parents = { revision_start: [] } revision_end = svnrepo.head_revision() self.log.info('[revision_start-revision_end]: [%s-%s]' % ( revision_start, revision_end)) - if revision_start == revision_end and revision_start is not 1: + if revision_start > revision_end and revision_start is not 1: self.log.info('%s@%s already injected.' % ( svnrepo.remote_url, revision_end)) - return {'status': True} + return { + 'eventful': False + } self.log.info('Processing %s.' % svnrepo) # process and store revision to swh (sent by by blocks of # 'revision_packet_size') - latest_rev = self.process_swh_revisions(svnrepo, - revision_start, - revision_end, - revision_parents) - self.process_swh_occurrence(latest_rev, origin) + try: + latest_rev = self.process_swh_revisions(svnrepo, + revision_start, + revision_end, + revision_parents) + except SvnLoaderException as e: + latest_rev = e.swh_revision + self.process_swh_occurrence(latest_rev, origin_visit) + self.process_swh_origin_visit(origin_visit, status='partial') + return { + 'eventful': True, + 'completion': 'partial', + 'state': { + 'origin': origin_visit['origin'], + 'revision': hashutil.hash_to_hex(latest_rev['id']) + } + } + else: + self.process_swh_occurrence(latest_rev, origin_visit) + self.process_swh_origin_visit(origin_visit, status='full') + return { + 'eventful': True, + 'completion': 'full', + } class SWHSvnLoader(BaseSvnLoader): """Swh svn loader is the main implementation destined for production. This implementation is able to deal with update on known svn repository. Default policy: It's to not add any information and be as close as possible from the svn data the server sent its way. The only thing that are added are the swh's revision 'extra_header' to be able to deal with update. """ def __init__(self, svn_url, destination_path, origin, svn_uuid=None): super().__init__(svn_url, destination_path, origin) self.svnrepo = svn.SWHSvnRepo( svn_url, origin['id'], self.storage, destination_path=destination_path, svn_uuid=svn_uuid) - def swh_previous_revision(self): + def swh_previous_revision(self, prev_swh_revision=None): """Retrieve swh's previous revision if any. """ - return self.svnrepo.swh_previous_revision() + return self.svnrepo.swh_previous_revision(prev_swh_revision) def check_history_not_altered(self, svnrepo, revision_start, swh_rev): """Given a svn repository, check if the history was not tampered with. """ revision_id = swh_rev['id'] parents = swh_rev['parents'] hash_data_per_revs = svnrepo.swh_hash_data_at_revision(revision_start) rev = revision_start rev, _, commit, objects_per_path = list(hash_data_per_revs)[0] dir_id = objects_per_path[b'']['checksums']['sha1_git'] swh_revision = self.build_swh_revision(rev, commit, dir_id, parents) swh_revision_id = git.compute_revision_sha1_git(swh_revision) return swh_revision_id == revision_id def build_swh_revision(self, rev, commit, dir_id, parents): """Build the swh revision dictionary. This adds: - the 'synthetic' flag to true - the 'extra_headers' containing the repository's uuid and the svn revision number. Args: rev: the svn revision commit: the commit metadata dir_id: the upper tree's hash identifier parents: the parents' identifiers Returns: The swh revision corresponding to the svn revision. """ return converters.build_swh_revision(rev, commit, self.svnrepo.uuid, dir_id, parents) def init_from(self, partial_swh_revision, previous_swh_revision): """Function to determine from where to start from. Args: - partial_swh_revision: A known revision from which the previous loading did not finish. - known_previous_revision: A known revision from which the previous loading did finish. Returns: The revision from which to start or None if nothing (fresh start). """ if partial_swh_revision and not previous_swh_revision: return partial_swh_revision if not partial_swh_revision and previous_swh_revision: return previous_swh_revision if partial_swh_revision and previous_swh_revision: # will determine from which to start from extra_headers1 = dict( partial_swh_revision['metadata']['extra_headers']) extra_headers2 = dict( previous_swh_revision['metadata']['extra_headers']) rev_start1 = int(extra_headers1['svn_revision']) rev_start2 = int(extra_headers2['svn_revision']) if rev_start1 <= rev_start2: return previous_swh_revision return partial_swh_revision return None - def process_repository(self, known_state=None): + def process_repository(self, origin_visit, last_known_swh_revision=None): svnrepo = self.svnrepo - origin = self.origin # default configuration revision_start = 1 revision_parents = { revision_start: [] } # Check if we already know a previous revision for that origin swh_rev = self.swh_previous_revision() # Determine from which known revision to start - swh_rev = self.init_from(known_state, previous_swh_revision=swh_rev) + swh_rev = self.init_from(last_known_swh_revision, + previous_swh_revision=swh_rev) if swh_rev: # Yes, we do. Try and update it. extra_headers = dict(swh_rev['metadata']['extra_headers']) revision_start = int(extra_headers['svn_revision']) revision_parents = { revision_start: swh_rev['parents'], } self.log.debug('svn export --ignore-keywords %s@%s' % ( svnrepo.remote_url, revision_start)) if swh_rev and not self.check_history_not_altered( svnrepo, revision_start, swh_rev): msg = 'History of svn %s@%s history modified. Skipping...' % ( # noqa svnrepo.remote_url, revision_start) self.log.warn(msg) - return {'status': False, 'stderr': msg} + return { + 'eventful': False, + 'status': 'failed', + } else: # now we know history is ok, we start at next revision revision_start = revision_start + 1 # and the parent become the latest know revision for # that repository revision_parents[revision_start] = [swh_rev['id']] revision_end = svnrepo.head_revision() self.log.info('[revision_start-revision_end]: [%s-%s]' % ( revision_start, revision_end)) if revision_start > revision_end and revision_start is not 1: self.log.info('%s@%s already injected.' % ( svnrepo.remote_url, revision_end)) - return {'status': True} + return { + 'eventful': False + } self.log.info('Processing %s.' % svnrepo) - # process and store revision to swh (sent by by blocks of - # 'revision_packet_size') - latest_rev = self.process_swh_revisions(svnrepo, - revision_start, - revision_end, - revision_parents) - self.process_swh_occurrence(latest_rev, origin) + try: + # process and store revision to swh (sent by by blocks of + # 'revision_packet_size') + latest_rev = self.process_swh_revisions(svnrepo, + revision_start, + revision_end, + revision_parents) + except SvnLoaderException as e: + latest_rev = e.swh_revision + self.process_swh_occurrence(latest_rev, origin_visit) + self.process_swh_origin_visit(origin_visit, status='full') + return { + 'eventful': True, + 'completion': 'partial', + 'state': { + 'origin': origin_visit['origin'], + 'revision': hashutil.hash_to_hex(latest_rev['id']) + } + } + else: + self.process_swh_occurrence(latest_rev, origin_visit) + self.process_swh_origin_visit(origin_visit, status='full') + return { + 'eventful': True, + 'completion': 'full', + } diff --git a/swh/loader/svn/svn.py b/swh/loader/svn/svn.py index 5cf5f66..9b7ce13 100644 --- a/swh/loader/svn/svn.py +++ b/swh/loader/svn/svn.py @@ -1,407 +1,422 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """SVN client in charge of iterating over svn logs and yield commit representations including the hash tree/content computations per svn commit. """ import os import tempfile import shutil from subvertpy.ra import RemoteAccess, Auth, get_username_provider from subvertpy import client, properties from swh.model import git from . import ra, utils, converters # When log message contains empty data DEFAULT_AUTHOR_MESSAGE = '' class SvnRepoException(ValueError): def __init__(self, svnrepo, e): super().__init__(e) self.svnrepo = svnrepo class BaseSvnRepo(): """Base svn repository representation for swh. To override some of the behavior regarding the message log properties, you can instantiate a subclass of this class and override: - def convert_commit_author(self, author) - def convert_commit_message(self, msg) - def convert_commit_date(self, date) cf. GitSvnSvnRepo, SwhSvnRepo for instanciation example. """ def __init__(self, remote_url, origin_id, storage, destination_path=None, svn_uuid=None): self.remote_url = remote_url.rstrip('/') self.storage = storage self.origin_id = origin_id if destination_path: os.makedirs(destination_path, exist_ok=True) root_dir = destination_path else: root_dir = '/tmp' self.local_dirname = tempfile.mkdtemp(suffix='.swh.loader', prefix='tmp.', dir=root_dir) local_name = os.path.basename(self.remote_url) auth = Auth([get_username_provider()]) # one connection for log iteration self.conn_log = RemoteAccess(self.remote_url, auth=auth) # another for replay self.conn = RemoteAccess(self.remote_url, auth=auth) # one client for update operation self.client = client.Client(auth=auth) self.local_url = os.path.join(self.local_dirname, local_name).encode( 'utf-8') if svn_uuid: self.uuid = svn_uuid.encode('utf-8') else: self.uuid = self.conn.get_uuid().encode('utf-8') def __str__(self): return str({'remote_url': self.remote_url, 'local_url': self.local_url, 'uuid': self.uuid, 'swh-origin': self.origin_id}) def head_revision(self): """Retrieve current revision of the repository's working copy. """ return self.conn.get_latest_revnum() def initial_revision(self): """Retrieve the initial revision from which the remote url appeared. Note: This should always be 1 since we won't be dealing with in-depth url. """ return 1 def convert_commit_message(self, msg): """Do something with message (e.g add extra line, etc...) cf. SvnRepo for a simple implementation. Args: msg (str): the commit message to convert. Returns: The transformed message as bytes. """ raise NotImplementedError('Should be overridden by subclass.') def convert_commit_date(self, date): """Convert the message date (e.g, convert into timestamp or whatever makes sense to you.). Args: date (str): the commit date to convert. Returns: The transformed date. """ raise NotImplementedError('Should be overridden by subclass.') def convert_commit_author(self, author): """Convert the commit author (e.g, convert into dict or whatever makes sense to you.). Args: author (str): the commit author to convert. Returns: The transformed author as dict. """ raise NotImplementedError('Should be overridden by subclass.') def __to_entry(self, log_entry): changed_paths, rev, revprops, has_children = log_entry author_date = self.convert_commit_date( revprops.get(properties.PROP_REVISION_DATE)) author = self.convert_commit_author( revprops.get(properties.PROP_REVISION_AUTHOR)) message = self.convert_commit_message( revprops.get(properties.PROP_REVISION_LOG, DEFAULT_AUTHOR_MESSAGE)) return { 'rev': rev, 'author_date': author_date, 'author_name': author, 'message': message, } def logs(self, revision_start, revision_end): """Stream svn logs between revision_start and revision_end by chunks of block_size logs. Yields revision and associated revision information between the revision start and revision_end. Args: revision_start: the svn revision starting bound revision_end: the svn revision ending bound Yields: tuple of revisions and logs. revisions: list of revisions in order logs: Dictionary with key revision number and value the log entry. The log entry is a dictionary with the following keys: - author_date: date of the commit - author_name: name of the author - message: commit message """ for log_entry in self.conn_log.iter_log(paths=None, start=revision_start, end=revision_end, discover_changed_paths=False): yield self.__to_entry(log_entry) def export(self, revision): """Export the repository to a given version. """ self.client.export(self.remote_url, to=self.local_url.decode('utf-8'), rev=revision, ignore_keywords=True) - def swh_previous_revision(self): + def swh_previous_revision(self, previous_swh_revision=None): """Look for possible existing revision in swh. + Args: + previous_swh_revision: (optional) id of a possible + previous swh revision + Returns: - The previous swh revision if found, None otherwise. + If previous_swh_revision is not None and do exists, + returns the complete instance. + Otherwise, check for a possible occurrence and returns the + targeted complete revision if it does exists. + Otherwise, returns None. """ storage = self.storage - occ = storage.occurrence_get(self.origin_id) - if occ: - revision_id = occ[0]['target'] - revisions = storage.revision_get([revision_id]) - - if revisions: - return revisions[0] + # got no previous revision, will check if some occurrence + # already exists for that origin + if not previous_swh_revision: + occ = storage.occurrence_get(self.origin_id) + if occ: + revision_id = occ[0]['target'] + revisions = storage.revision_get([revision_id]) + + if revisions: + return revisions[0] + else: + revs = storage.revision_get([previous_swh_revision]) + if revs: + return revs[0] def swh_hash_data_per_revision(self, start_revision, end_revision): """Compute swh hash data per each revision between start_revision and end_revision. Args: start_revision: starting revision end_revision: ending revision Yields: tuple (rev, nextrev, commit, objects_per_path) - rev: current revision - nextrev: next revision - commit: commit data (author, date, message) for such revision - objects_per_path: dictionary of path, swh hash data with type """ hashes = {} for commit in self.logs(start_revision, end_revision): rev = commit['rev'] hashes = self.swhreplay.compute_hashes(rev) if rev == end_revision: nextrev = None else: nextrev = rev + 1 yield rev, nextrev, commit, hashes def swh_hash_data_at_revision(self, revision): """Compute the hash data at revision. Expected to be used for update only. """ # Update the disk at revision self.export(revision) # Compute the current hashes on disk hashes = git.compute_hashes_from_directory(self.local_url) hashes = utils.convert_hashes_with_relative_path( hashes, rootpath=self.local_url) # Update the replay collaborator with the right state self.swhreplay = ra.SWHReplay( conn=self.conn, rootpath=self.local_url, objects=hashes) # Retrieve the commit information for revision commit = list(self.logs(revision, revision))[0] yield revision, revision + 1, commit, hashes def clean_fs(self): """Clean up the local working copy. """ shutil.rmtree(self.local_dirname) class GitSvnSvnRepo(BaseSvnRepo): """Svn repository mapping a-la git-svn. This class does exactly as BaseSvnRepo except for: - the commit message which is extended with a new line and then encoded - the commit author is converted using the repository's uuid for his/her email - the commit date is transformed into timestamp and truncated - Extra commit line in commit message. - user@ in raw commit message - truncated timestamp in raw commit message """ def __init__(self, remote_url, origin_id, storage, destination_path=None, svn_uuid=None): super().__init__(remote_url, origin_id, storage, destination_path=destination_path, svn_uuid=svn_uuid) self.swhreplay = ra.SWHReplayNoEmptyFolder( conn=self.conn, rootpath=self.local_url) self.with_empty_folder = False def convert_commit_message(self, msg): """Add an extra line to the commit message and encode it in utf-8. Args: msg (str): the commit message to convert. Returns: The transformed message as bytes. """ return ('%s\n' % msg).encode('utf-8') def convert_commit_date(self, date): """Convert the commit message date into truncated timestamp in swh format. Args: date (str): the commit date to convert. Returns: The transformed date. """ return converters.svn_date_to_gitsvn_date(date) def convert_commit_author(self, author): """Convert the commit author into an swh person. The user becomes a dictionary of the form: { name: author, email: author@repo-uuid, fullname: author } If the user is already some kind of fullname, this is what is used as fullname. Args: author (str): the commit author to convert. Returns: The transformed author as dict. """ return converters.svn_author_to_gitsvn_person(author, self.uuid) class SWHSvnRepo(BaseSvnRepo): """This class does exactly as BaseSvnRepo except for: - the commit message which is simply encoded - the commit author is left as is. - the commit timestamp is left as is. """ def __init__(self, remote_url, origin_id, storage, destination_path=None, svn_uuid=None): super().__init__(remote_url, origin_id, storage, destination_path=destination_path, svn_uuid=svn_uuid) self.swhreplay = ra.SWHReplay( conn=self.conn, rootpath=self.local_url) def convert_commit_message(self, msg): """Simply encode the commit message. Args: msg (str): the commit message to convert. Returns: The transformed message as bytes. """ return msg.encode('utf-8') def convert_commit_date(self, date): """Convert the message commit date into a timestamp in swh format. The precision is kept. Args: date (str): the commit date to convert. Returns: The transformed date. """ return converters.svn_date_to_swh_date(date) def convert_commit_author(self, author): """Convert the commit author into an swh person. The user becomes a dictionary of the form: { name: author, email: '', fullname: author } Args: author (str): the commit author to convert. Returns: The transformed author as dict. """ return converters.svn_author_to_swh_person(author) diff --git a/swh/loader/svn/tasks.py b/swh/loader/svn/tasks.py index 46685a9..f4a473d 100644 --- a/swh/loader/svn/tasks.py +++ b/swh/loader/svn/tasks.py @@ -1,131 +1,107 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from swh.loader.core import tasks -from swh.scheduler.backend import OneShotSchedulerBackend - -from .loader import GitSvnSvnLoader, SWHSvnLoader, SvnLoaderException -from .loader import converters - +import datetime -class ReportToSchedulerWhenFail: - """Mixin to permit to enhance LoadSvnRepositoryTsk with scheduler - backend dependency. +from swh.core import hashutil +from swh.loader.core import tasks - """ - def __init__(self): - super().__init__() - self.scheduler_backend = OneShotSchedulerBackend() +from .loader import GitSvnSvnLoader, SWHSvnLoader -class LoadSvnRepositoryTsk(ReportToSchedulerWhenFail, tasks.LoaderCoreTask): +class LoadSvnRepositoryTsk(tasks.LoaderCoreTask): """Import one svn repository to Software Heritage. """ CONFIG_BASE_FILENAME = 'loader/svn.ini' ADDITIONAL_CONFIG = { 'storage_class': ('str', 'remote_storage'), 'storage_args': ('list[str]', ['http://localhost:5000/']), 'with_policy': ('string', 'swh'), # Default, other possible # value is 'gitsvn' } task_queue = 'swh_loader_svn' def run(self, *args, **kwargs): """Import a svn repository. Args: - svn_url: svn's repository url - destination_path: root directory to locally retrieve svn's data - swh_revision: Optional extra swh revision to start from. cf. swh.loader.svn.SvnLoader.process docstring """ destination_path = kwargs['destination_path'] # local svn url svn_url = kwargs['svn_url'] # if original_svn_url is mentioned, this means we load a local mirror original_svn_url = kwargs.get('original_svn_url') # potential uuid overwrite original_svn_uuid = kwargs.get('original_svn_uuid') # Make sure we have all that's needed if original_svn_url and not original_svn_uuid: msg = "When loading a local mirror, you must specify the original repository's uuid." # noqa self.log.error('%s. Skipping mirror %s' % (msg, svn_url)) return # Determine the origin url origin_url = original_svn_url if original_svn_url else svn_url if 'origin' not in kwargs: # first time, we'll create the origin origin = { - 'type': 'svn', 'url': origin_url, + 'type': 'svn', } origin['id'] = self.storage.origin_add_one(origin) - retry = False else: origin = { 'id': kwargs['origin'], 'url': origin_url, 'type': 'svn' } - retry = True - - fetch_history_id = self.open_fetch_history(origin['id']) - - try: - # Determine which loader to trigger - if self.config['with_policy'] == 'gitsvn': - # this one compute hashes but do not store anywhere - loader = GitSvnSvnLoader(svn_url, destination_path, origin, - svn_uuid=original_svn_uuid) - elif self.config['with_policy'] == 'swh': - # the real production use case with storage and all - loader = SWHSvnLoader(svn_url, destination_path, origin, - svn_uuid=original_svn_uuid) - else: - raise ValueError('Only gitsvn or swh policies are supported in' - '\'with_policy\' entry. ' - 'Please adapt your svn.ini file accordingly') - if retry: - swh_revision = converters.scheduler_to_loader_revision( - kwargs['swh_revision']) - else: - swh_revision = None - - result = loader.load(swh_revision) - except SvnLoaderException as e: - # Reschedule a task if possible - if retry and not e.swh_revision: - swh_rev = swh_revision - else: - swh_rev = e.swh_revision - - swh_rev = converters.loader_to_scheduler_revision(swh_rev) - - self.scheduler_backend.create_task({ - 'type': 'svn-loader', - 'arguments': { - 'args': None, - 'kwargs': { - 'origin': origin['id'], - 'svn_url': svn_url, - 'original_svn_url': original_svn_url, - 'original_svn_uuid': original_svn_uuid, - 'destination_path': destination_path, - 'swh_revision': swh_rev, - 'error': str(e), - } - } + + date_visit = datetime.datetime.now(tz=datetime.timezone.utc) + origin_visit = self.storage.origin_visit_add(origin['id'], + date_visit) + + origin_visit.update({ + 'date': date_visit + }) + + # Determine which loader to trigger + if self.config['with_policy'] == 'gitsvn': + # this one compute hashes but do not store anywhere + loader = GitSvnSvnLoader(svn_url, destination_path, origin, + svn_uuid=original_svn_uuid) + elif self.config['with_policy'] == 'swh': + # the real production use case with storage and all + loader = SWHSvnLoader(svn_url, destination_path, origin, + svn_uuid=original_svn_uuid) + else: + raise ValueError('Only gitsvn or swh policies are supported in' + '\'with_policy\' entry. ' + 'Please adapt your svn.ini file accordingly') + if 'swh_revision' in kwargs: + swh_revision = hashutil.hex_to_hash(kwargs['swh_revision']) + else: + swh_revision = None + + result = loader.load(origin_visit, swh_revision) + + # Check for partial completion to complete state data + if 'completion' in result and result['completion'] == 'partial': + state = result['state'] + state.update({ + 'svn_url': svn_url, + 'original_svn_url': origin_url, + 'original_svn_uuid': original_svn_uuid, }) - self.log.error( - 'Error during loading: %s - Svn repository rescheduled.' % e) - result = {'status': False} + result['state'] = state - self.close_fetch_history(fetch_history_id, result) + return result diff --git a/swh/loader/svn/tests/test_converters.py b/swh/loader/svn/tests/test_converters.py index 0093fa8..d2e28b5 100644 --- a/swh/loader/svn/tests/test_converters.py +++ b/swh/loader/svn/tests/test_converters.py @@ -1,320 +1,321 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest from nose.tools import istest from swh.loader.svn import converters class TestAuthorGitSvnConverters(unittest.TestCase): @istest def svn_author_to_gitsvn_person(self): """The author should have name, email and fullname filled. """ actual_person = converters.svn_author_to_gitsvn_person( 'tony ', repo_uuid=None) self.assertEquals(actual_person, { 'fullname': b'tony ', 'name': b'tony', 'email': b'ynot@dagobah', }) @istest def svn_author_to_gitsvn_person_no_email(self): """The author should see his/her email filled with author@. """ actual_person = converters.svn_author_to_gitsvn_person( 'tony', repo_uuid=b'some-uuid') self.assertEquals(actual_person, { 'fullname': b'tony ', 'name': b'tony', 'email': b'tony@some-uuid', }) @istest def svn_author_to_gitsvn_person_empty_person(self): """The empty person should see name, fullname and email filled. """ actual_person = converters.svn_author_to_gitsvn_person( '', repo_uuid=b'some-uuid') self.assertEqual(actual_person, { 'fullname': b'(no author) <(no author)@some-uuid>', 'name': b'(no author)', 'email': b'(no author)@some-uuid' }) class TestAuthorSWHConverters(unittest.TestCase): @istest def svn_author_to_swh_person(self): """The author should have name, email and fullname filled. """ actual_person = converters.svn_author_to_swh_person( 'tony ') self.assertEquals(actual_person, { 'fullname': b'tony ', 'name': b'tony', 'email': b'ynot@dagobah', }) @istest def svn_author_to_swh_person_no_email(self): """The author and fullname should be the same as the input (author). """ actual_person = converters.svn_author_to_swh_person('tony') self.assertEquals(actual_person, { 'fullname': b'tony', 'name': b'tony', 'email': None, }) @istest def svn_author_to_swh_person_empty_person(self): """Empty person has only its fullname filled with the empty byte-string. """ actual_person = converters.svn_author_to_swh_person('') self.assertEqual(actual_person, { 'fullname': b'', 'name': None, 'email': None, }) class TestSWHRevisionConverters(unittest.TestCase): @istest def build_swh_revision_default(self): """This should build the swh revision with the swh revision's extra headers about the repository. """ actual_swh_revision = converters.build_swh_revision( repo_uuid=b'uuid', dir_id='dir-id', commit={ 'author_name': { 'name': b'theo', 'email': b'theo@uuid', 'fullname': b'theo ' }, 'message': b'commit message', 'author_date': { 'timestamp': 1088108379, 'offset': 0 } }, rev=10, parents=['123']) date = {'timestamp': 1088108379, 'offset': 0} self.assertEquals(actual_swh_revision, { 'date': date, 'committer_date': date, 'type': 'svn', 'directory': 'dir-id', 'message': b'commit message', 'author': { 'name': b'theo', 'email': b'theo@uuid', 'fullname': b'theo ' }, 'committer': { 'name': b'theo', 'email': b'theo@uuid', 'fullname': b'theo ' }, 'synthetic': True, 'metadata': { 'extra_headers': [ ['svn_repo_uuid', b'uuid'], ['svn_revision', b'10'], ] }, 'parents': ['123'], }) class TestGitSvnRevisionConverters(unittest.TestCase): @istest def build_gitsvn_swh_revision_default(self): """This should build the swh revision without the swh revision's extra headers about the repository. """ actual_swh_revision = converters.build_gitsvn_swh_revision( dir_id='dir-id', commit={ 'author_name': { 'name': b'theo', 'email': b'theo@uuid', 'fullname': b'theo ' }, 'message': b'commit message', 'author_date': { 'timestamp': 1088108379, 'offset': 0 } }, rev=10, parents=['123']) date = {'timestamp': 1088108379, 'offset': 0} self.assertEquals(actual_swh_revision, { 'date': date, 'committer_date': date, 'type': 'svn', 'directory': 'dir-id', 'message': b'commit message', 'author': { 'name': b'theo', 'email': b'theo@uuid', 'fullname': b'theo ' }, 'committer': { 'name': b'theo', 'email': b'theo@uuid', 'fullname': b'theo ' }, 'synthetic': True, 'metadata': None, 'parents': ['123'], }) class TestSWHOccurrence(unittest.TestCase): @istest def build_swh_occurrence(self): actual_occ = converters.build_swh_occurrence('revision-id', 'origin-id', - 'some-date') + visit=10) self.assertEquals(actual_occ, { 'branch': 'master', 'target': 'revision-id', 'target_type': 'revision', 'origin': 'origin-id', - 'date': 'some-date'}) + 'visit': 10 + }) class ConvertSWHDate(unittest.TestCase): @istest def svn_date_to_swh_date(self): """The timestamp should not be tampered with and include the decimals. """ self.assertEquals( converters.svn_date_to_swh_date('2011-05-31T06:04:39.500900Z'), { 'timestamp': 1306821879.5009, 'offset': 0 }) self.assertEquals( converters.svn_date_to_swh_date('2011-05-31T06:04:39.800722Z'), { 'timestamp': 1306821879.800722, 'offset': 0 }) @istest def svn_date_to_swh_date_epoch(self): """Empty date should be EPOCH (timestamp and offset at 0).""" # It should return 0, epoch self.assertEquals({'timestamp': 0, 'offset': 0}, converters.svn_date_to_swh_date('')) self.assertEquals({'timestamp': 0, 'offset': 0}, converters.svn_date_to_swh_date(None)) class ConvertGitSvnDate(unittest.TestCase): @istest def svn_date_to_gitsvn_date(self): """The timestamp should be truncated to be an integer.""" actual_ts = converters.svn_date_to_gitsvn_date( '2011-05-31T06:04:39.800722Z') self.assertEquals(actual_ts, {'timestamp': 1306821879, 'offset': 0}) @istest def svn_date_to_gitsvn_date_epoch(self): """Empty date should be EPOCH (timestamp and offset at 0).""" # It should return 0, epoch self.assertEquals({'timestamp': 0, 'offset': 0}, converters.svn_date_to_gitsvn_date('')) self.assertEquals({'timestamp': 0, 'offset': 0}, converters.svn_date_to_gitsvn_date(None)) class ConvertSWHRevision(unittest.TestCase): @istest def loader_to_scheduler_revision(self): actual_rev = converters.loader_to_scheduler_revision({ 'parents': [b'e\n\xbe\xe9\xc0\x87y\xfeG\xf7\xcfG\x82h\xa8i\xe8\xfe\xe2\x13'], # noqa 'id': b'\xedd\x92w\xab\xb2\x16,\xea*\x90O8\x0f\x96/\xfb\xd4\x16`', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', b'bc7d6c17-68a5-4917-9c54-c565d7424229'], ['svn_revision', b'4'] ] } }) self.assertEquals(actual_rev, { 'id': 'ed649277abb2162cea2a904f380f962ffbd41660', 'parents': ['650abee9c08779fe47f7cf478268a869e8fee213'], 'metadata': { 'extra_headers': [ ['svn_repo_uuid', 'bc7d6c17-68a5-4917-9c54-c565d7424229'], ['svn_revision', '4'] ] } }) @istest def loader_to_scheduler_revision_none(self): self.assertIsNone(converters.loader_to_scheduler_revision(None)) @istest def scheduler_to_loader_revision(self): actual_rev = converters.scheduler_to_loader_revision({ 'id': 'ed649277abb2162cea2a904f380f962ffbd41660', 'parents': ['650abee9c08779fe47f7cf478268a869e8fee213'], 'metadata': { 'extra_headers': [ ['svn_repo_uuid', 'bc7d6c17-68a5-4917-9c54-c565d7424229'], ['svn_revision', '4'] ] } }) self.assertEquals(actual_rev, { 'parents': [b'e\n\xbe\xe9\xc0\x87y\xfeG\xf7\xcfG\x82h\xa8i\xe8\xfe\xe2\x13'], # noqa 'id': b'\xedd\x92w\xab\xb2\x16,\xea*\x90O8\x0f\x96/\xfb\xd4\x16`', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', 'bc7d6c17-68a5-4917-9c54-c565d7424229'], ['svn_revision', '4'] ] } }) @istest def scheduler_to_loader_revision_none(self): self.assertIsNone(converters.scheduler_to_loader_revision(None)) diff --git a/swh/loader/svn/tests/test_loader.py b/swh/loader/svn/tests/test_loader.py index 033e2ad..7157707 100644 --- a/swh/loader/svn/tests/test_loader.py +++ b/swh/loader/svn/tests/test_loader.py @@ -1,587 +1,634 @@ # Copyright (C) 2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from nose.tools import istest from swh.core import hashutil from swh.loader.svn.loader import GitSvnSvnLoader, SWHSvnLoader from test_base import BaseTestSvnLoader # Define loaders with no storage # They'll just accumulate the data in place # Only for testing purposes. class TestSvnLoader: """Mixin class to inhibit the persistence and keep in memory the data sent for storage. cf. GitSvnLoaderNoStorage, SWHSvnLoaderNoStorage """ def __init__(self, svn_url, destination_path, origin): super().__init__(svn_url, destination_path, origin) # We don't want to persist any result in this test context self.config['send_contents'] = False self.config['send_directories'] = False self.config['send_revisions'] = False self.config['send_releases'] = False self.config['send_occurrences'] = False # Init the state self.all_contents = [] self.all_directories = [] self.all_revisions = [] self.all_releases = [] self.all_occurrences = [] def maybe_load_contents(self, all_contents): self.all_contents.extend(all_contents) def maybe_load_directories(self, all_directories): self.all_directories.extend(all_directories) def maybe_load_revisions(self, all_revisions): self.all_revisions.extend(all_revisions) def maybe_load_releases(self, releases): raise ValueError('If called, the test must break.') def maybe_load_occurrences(self, all_occurrences): self.all_occurrences.extend(all_occurrences) + def process_swh_origin_visit(self, origin_visit, status): + # Do nothing during origin_visit update + pass + class GitSvnLoaderNoStorage(TestSvnLoader, GitSvnSvnLoader): """A GitSvnLoader with no persistence. Context: Load an svn repository using the git-svn policy. """ def __init__(self, svn_url, destination_path, origin): super().__init__(svn_url, destination_path, origin) class SWHSvnLoaderNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: Load a new svn repository using the swh policy (so no update). """ - def swh_previous_revision(self): + def swh_previous_revision(self, prev_swh_revision=None): """We do not know this repository so no revision. """ return None class SWHSvnLoaderUpdateNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: Load a known svn repository using the swh policy. We can either: - do nothing since it does not contain any new commit (so no change) - either check its history is not altered and update in consequence by loading the new revision """ - def swh_previous_revision(self): + def swh_previous_revision(self, prev_swh_revision=None): """Avoid the storage persistence call and return the expected previous revision for that repository. Check the following for explanation about the hashes: - test_loader.org for (swh policy). - cf. SWHSvnLoaderITTest """ return { 'id': hashutil.hex_to_hash( '4876cb10aec6f708f7466dddf547567b65f6c39c'), 'parents': [hashutil.hex_to_hash( 'a3a577948fdbda9d1061913b77a1588695eadb41')], 'directory': hashutil.hex_to_hash( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '6'] ] } } class SWHSvnLoaderUpdateHistoryAlteredNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: Load a known svn repository using the swh policy with its history altered so we do not update it. """ - def swh_previous_revision(self): + def swh_previous_revision(self, prev_swh_revision=None): """Avoid the storage persistence call and return the expected previous revision for that repository. Check the following for explanation about the hashes: - test_loader.org for (swh policy). - cf. SWHSvnLoaderITTest """ return { # Changed the revision id's hash to simulate history altered 'id': hashutil.hex_to_hash( 'badbadbadbadf708f7466dddf547567b65f6c39d'), 'parents': [hashutil.hex_to_hash( 'a3a577948fdbda9d1061913b77a1588695eadb41')], 'directory': hashutil.hex_to_hash( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', b'6'] ] } } class GitSvnLoaderITTest(BaseTestSvnLoader): def setUp(self): super().setUp() self.origin = {'id': 1, 'type': 'svn', 'url': 'file:///dev/null'} + self.origin_visit = { + 'origin': self.origin['id'], + 'visit': 1, + } + self.loader = GitSvnLoaderNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """Process a repository with gitsvn policy should be ok.""" # when - self.loader.process_repository() + self.loader.process_repository(self.origin_visit) # then self.assertEquals(len(self.loader.all_revisions), 6) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 1) last_revision = 'bad4a83737f337d47e0ba681478214b07a707218' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash # noqa '22c0fa5195a53f2e733ec75a9b6e9d1624a8b771': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', # noqa '17a631d474f49bbebfdf3d885dcde470d7faafd7': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', # noqa 'c8a9172b2a615d461154f61158180de53edc6070': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', # noqa '7c8f83394b6e8966eb46f0d3416c717612198a4b': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', # noqa '852547b3b2bb76c8582cee963e8aa180d552a15c': 'ab047e38d1532f61ff5c3621202afc3e763e9945', # noqa last_revision: '9bcfc25001b71c333b4b5a89224217de81c56e2e', # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) occ = self.loader.all_occurrences[0] self.assertEquals(hashutil.hash_to_hex(occ['target']), last_revision) self.assertEquals(occ['origin'], self.origin['id']) class SWHSvnLoaderNewRepositoryITTest(BaseTestSvnLoader): def setUp(self): super().setUp() self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} + self.origin_visit = { + 'origin': self.origin['id'], + 'visit': 2, + } + self.loader = SWHSvnLoaderNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """Process a new repository with swh policy should be ok. """ # when - self.loader.process_repository() + self.loader.process_repository(self.origin_visit) # then self.assertEquals(len(self.loader.all_revisions), 6) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 1) last_revision = '4876cb10aec6f708f7466dddf547567b65f6c39c' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71': '669a71cce6c424a81ba42b7dc5d560d32252f0ca', # noqa '95edacc8848369d6fb1608e887d6d2474fd5224f': '008ac97a1118560797c50e3392fa1443acdaa349', # noqa 'fef26ea45a520071711ba2b9d16a2985ee837021': '3780effbe846a26751a95a8c95c511fb72be15b4', # noqa '3f51abf3b3d466571be0855dfa67e094f9ceff1b': 'ffcca9b09c5827a6b8137322d4339c8055c3ee1e', # noqa 'a3a577948fdbda9d1061913b77a1588695eadb41': '7dc52cc04c3b8bd7c085900d60c159f7b846f866', # noqa last_revision: '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) occ = self.loader.all_occurrences[0] self.assertEquals(hashutil.hash_to_hex(occ['target']), last_revision) self.assertEquals(occ['origin'], self.origin['id']) class SWHSvnLoaderUpdateWithNoChangeITTest(BaseTestSvnLoader): def setUp(self): super().setUp() self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} + self.origin_visit = { + 'origin': self.origin['id'], + 'visit': 3, + } + self.loader = SWHSvnLoaderUpdateNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """Process a known repository with swh policy and no new data should be ok. """ # when - self.loader.process_repository() + self.loader.process_repository(self.origin_visit) # then self.assertEquals(len(self.loader.all_revisions), 0) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 0) class SWHSvnLoaderUpdateWithHistoryAlteredITTest(BaseTestSvnLoader): def setUp(self): # the svn repository pkg-gourmet has been updated with changes super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} + self.origin_visit = { + 'origin': self.origin['id'], + 'visit': 4, + } + self.loader = SWHSvnLoaderUpdateHistoryAlteredNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """Process a known repository with swh policy and history altered should stop and do nothing. """ # when - self.loader.process_repository() + self.loader.process_repository(self.origin_visit) # then # we got the previous run's last revision (rev 6) # so 2 news + 1 old self.assertEquals(len(self.loader.all_revisions), 0) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 0) class SWHSvnLoaderUpdateWithChangesITTest(BaseTestSvnLoader): def setUp(self): # the svn repository pkg-gourmet has been updated with changes super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} + self.origin_visit = { + 'origin': self.origin['id'], + 'visit': 5, + } + self.loader = SWHSvnLoaderUpdateNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """Process a known repository with swh policy and new data should yield new revisions and occurrence. """ # when - self.loader.process_repository() + self.loader.process_repository(self.origin_visit) # then # we got the previous run's last revision (rev 6) # so 2 new self.assertEquals(len(self.loader.all_revisions), 5) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 1) last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) occ = self.loader.all_occurrences[0] self.assertEquals(hashutil.hash_to_hex(occ['target']), last_revision) self.assertEquals(occ['origin'], self.origin['id']) class SWHSvnLoaderUpdateWithUnfinishedLoadingChangesITTest(BaseTestSvnLoader): def setUp(self): super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} + self.origin_visit = { + 'origin': self.origin['id'], + 'visit': 6 + } + self.loader = SWHSvnLoaderNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """Process a known repository with swh policy, the previous run did not finish, so this finishes the loading """ previous_unfinished_revision = { 'id': hashutil.hex_to_hash( '4876cb10aec6f708f7466dddf547567b65f6c39c'), 'parents': [hashutil.hex_to_hash( 'a3a577948fdbda9d1061913b77a1588695eadb41')], 'directory': hashutil.hex_to_hash( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '6'] ] } } # when self.loader.process_repository( - known_state=previous_unfinished_revision) + self.origin_visit, + last_known_swh_revision=previous_unfinished_revision) # then # we got the previous run's last revision (rev 6) # so 2 new self.assertEquals(len(self.loader.all_revisions), 5) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 1) last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) occ = self.loader.all_occurrences[0] self.assertEquals(hashutil.hash_to_hex(occ['target']), last_revision) self.assertEquals(occ['origin'], self.origin['id']) class SWHSvnLoaderUpdateWithUnfinishedLoadingChangesButOccurrenceDoneITTest( BaseTestSvnLoader): def setUp(self): super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} + self.origin_visit = { + 'origin': self.origin['id'], + 'visit': 9, + } + self.loader = SWHSvnLoaderUpdateNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """known repository, swh policy, unfinished revision is less recent than occurrence, we start from last occurrence. """ previous_unfinished_revision = { 'id': hashutil.hex_to_hash( 'a3a577948fdbda9d1061913b77a1588695eadb41'), 'parents': [hashutil.hex_to_hash( '3f51abf3b3d466571be0855dfa67e094f9ceff1b')], 'directory': hashutil.hex_to_hash( '7dc52cc04c3b8bd7c085900d60c159f7b846f866'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '5'] ] } } # when self.loader.process_repository( - known_state=previous_unfinished_revision) + self.origin_visit, + last_known_swh_revision=previous_unfinished_revision) # then # we got the previous run's last revision (rev 6) # so 2 new self.assertEquals(len(self.loader.all_revisions), 5) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 1) last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) occ = self.loader.all_occurrences[0] self.assertEquals(hashutil.hash_to_hex(occ['target']), last_revision) self.assertEquals(occ['origin'], self.origin['id']) class SWHSvnLoaderUpdateLessRecentNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: Load a known svn repository using the swh policy. The last occurrence seen is less recent than a previous unfinished crawl. """ - def swh_previous_revision(self): + def swh_previous_revision(self, prev_swh_revision=None): """Avoid the storage persistence call and return the expected previous revision for that repository. Check the following for explanation about the hashes: - test_loader.org for (swh policy). - cf. SWHSvnLoaderITTest """ return { 'id': hashutil.hex_to_hash( 'a3a577948fdbda9d1061913b77a1588695eadb41'), 'parents': [hashutil.hex_to_hash( '3f51abf3b3d466571be0855dfa67e094f9ceff1b')], 'directory': hashutil.hex_to_hash( '7dc52cc04c3b8bd7c085900d60c159f7b846f866'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '5'] ] } } class SWHSvnLoaderUnfinishedLoadingChangesSinceLastOccurrenceITTest( BaseTestSvnLoader): def setUp(self): super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} + self.origin_visit = { + 'origin': self.origin['id'], + 'visit': 1, + } + self.loader = SWHSvnLoaderUpdateLessRecentNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """known repository, swh policy, unfinished revision is less recent than occurrence, we start from last occurrence. """ previous_unfinished_revision = { 'id': hashutil.hex_to_hash( '4876cb10aec6f708f7466dddf547567b65f6c39c'), 'parents': [hashutil.hex_to_hash( 'a3a577948fdbda9d1061913b77a1588695eadb41')], 'directory': hashutil.hex_to_hash( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '6'] ] } } # when self.loader.process_repository( - known_state=previous_unfinished_revision) + self.origin_visit, + last_known_swh_revision=previous_unfinished_revision) # then # we got the previous run's last revision (rev 6) # so 2 new self.assertEquals(len(self.loader.all_revisions), 5) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 1) last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) occ = self.loader.all_occurrences[0] self.assertEquals(hashutil.hash_to_hex(occ['target']), last_revision) self.assertEquals(occ['origin'], self.origin['id'])