diff --git a/PKG-INFO b/PKG-INFO index 754511a..52afbb2 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.loader.svn -Version: 0.0.24 +Version: 0.0.25 Summary: Software Heritage Loader SVN Home-page: https://forge.softwareheritage.org/diffusion/DLDSVN Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/debian/control b/debian/control index 2233960..87b427c 100644 --- a/debian/control +++ b/debian/control @@ -1,35 +1,35 @@ Source: swh-loader-svn Maintainer: Software Heritage developers Section: python Priority: optional Build-Depends: debhelper (>= 9), dh-python, python3-all, python3-click, python3-dateutil, python3-nose, python3-retrying, python3-setuptools, python3-subvertpy (>= 0.9.4~), python3-swh.core (>= 0.0.19~), - python3-swh.loader.core (>= 0.0.13~), - python3-swh.model (>= 0.0.11~), + python3-swh.loader.core (>= 0.0.14~), + python3-swh.model (>= 0.0.15~), python3-swh.scheduler (>= 0.0.11~), - python3-swh.storage (>= 0.0.79~), + python3-swh.storage (>= 0.0.83~), python3-vcversioner Standards-Version: 3.9.6 Homepage: https://forge.softwareheritage.org/diffusion/DLDSVN/ Package: python3-swh.loader.svn Architecture: all -Depends: pigz, +Depends: gzip, python3-swh.core (>= 0.0.19~), - python3-swh.loader.core (>= 0.0.13~), - python3-swh.model (>= 0.0.11~), + python3-swh.loader.core (>= 0.0.14~), + python3-swh.model (>= 0.0.15~), python3-swh.scheduler (>= 0.0.11~), - python3-swh.storage (>= 0.0.79~), + python3-swh.storage (>= 0.0.83~), subversion, ${misc:Depends}, ${python3:Depends} Description: Software Heritage Loader Svn Module in charge of loading svn repositories into swh storage. diff --git a/requirements-swh.txt b/requirements-swh.txt index e7b3fc5..1a818ed 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,5 +1,5 @@ swh.core >= 0.0.19 -swh.storage >= 0.0.79 -swh.model >= 0.0.11 +swh.storage >= 0.0.83 +swh.model >= 0.0.15 swh.scheduler >= 0.0.11 -swh.loader.core >= 0.0.13 +swh.loader.core >= 0.0.14 diff --git a/swh.loader.svn.egg-info/PKG-INFO b/swh.loader.svn.egg-info/PKG-INFO index 754511a..52afbb2 100644 --- a/swh.loader.svn.egg-info/PKG-INFO +++ b/swh.loader.svn.egg-info/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.loader.svn -Version: 0.0.24 +Version: 0.0.25 Summary: Software Heritage Loader SVN Home-page: https://forge.softwareheritage.org/diffusion/DLDSVN Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/swh.loader.svn.egg-info/requires.txt b/swh.loader.svn.egg-info/requires.txt index 457a2a3..f69bdec 100644 --- a/swh.loader.svn.egg-info/requires.txt +++ b/swh.loader.svn.egg-info/requires.txt @@ -1,10 +1,10 @@ click python-dateutil retrying subvertpy>=0.9.4 swh.core>=0.0.19 -swh.loader.core>=0.0.13 -swh.model>=0.0.11 +swh.loader.core>=0.0.14 +swh.model>=0.0.15 swh.scheduler>=0.0.11 -swh.storage>=0.0.79 +swh.storage>=0.0.83 vcversioner diff --git a/swh/loader/svn/converters.py b/swh/loader/svn/converters.py index d0cbe48..0803292 100644 --- a/swh/loader/svn/converters.py +++ b/swh/loader/svn/converters.py @@ -1,242 +1,196 @@ -# Copyright (C) 2015-2016 The Software Heritage developers +# Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from email import utils -from swh.core import hashutil - from .utils import strdate_to_timestamp def svn_date_to_gitsvn_date(strdate): """Convert a string date to an swh one. Args: strdate: A string formatted for .utils.strdate_to_timestamp to do its jobs Returns: An swh date format with an integer timestamp. """ ts = strdate_to_timestamp(strdate) return { 'timestamp': { 'seconds': ts['seconds'], 'microseconds': 0, }, 'offset': 0 } def svn_date_to_swh_date(strdate): """Convert a string date to an swh one. Args: strdate: A string formatted for .utils.strdate_to_timestamp to do its jobs Returns: An swh date format """ return { 'timestamp': strdate_to_timestamp(strdate), 'offset': 0 } def svn_author_to_swh_person(author): """Convert an svn author to an swh person. Default policy: No information is added. Args: author (string): the svn author (in bytes) Returns: a dictionary with keys: fullname: the author's associated fullname name: the author's associated name email: None (no email in svn) """ if not author: return {'fullname': b'', 'name': None, 'email': None} author = author.encode('utf-8') if b'<' in author and b'>' in author: name, email = utils.parseaddr(author.decode('utf-8')) return { 'fullname': author, 'name': name.encode('utf-8'), 'email': email.encode('utf-8') } return {'fullname': author, 'email': None, 'name': author} def svn_author_to_gitsvn_person(author, repo_uuid): """Convert an svn author to a person suitable for insertion. Default policy: If no email is found, the email is created using the author and the repo_uuid. Args: author (string): the svn author (in bytes) repo_uuid (bytes): the repository's uuid Returns: a dictionary with keys: fullname: the author's associated fullname name: the author's associated name email: None (no email in svn) """ if not author: author = '(no author)' author = author.encode('utf-8') if b'<' in author and b'>' in author: name, email = utils.parseaddr(author.decode('utf-8')) return { 'fullname': author, 'name': name.encode('utf-8'), 'email': email.encode('utf-8') } # we'll construct the author's fullname the same way git svn does # 'user ' email = b'@'.join([author, repo_uuid]) return { 'fullname': b''.join([author, b' ', b'<', email, b'>']), 'name': author, 'email': email, } def build_swh_revision(rev, commit, repo_uuid, dir_id, parents): """Given a svn revision, build a swh revision. This adds an ['metadata']['extra-headers'] entry with the repository's uuid and the svn revision. Args: - rev: the svn revision number - commit: the commit metadata - repo_uuid: The repository's uuid - dir_id: the tree's hash identifier - parents: the revision's parents identifier Returns: The swh revision dictionary. """ author = commit['author_name'] msg = commit['message'] date = commit['author_date'] metadata = { 'extra_headers': [ ['svn_repo_uuid', repo_uuid], ['svn_revision', str(rev).encode('utf-8')] ] } return { 'date': date, 'committer_date': date, 'type': 'svn', 'directory': dir_id, 'message': msg, 'author': author, 'committer': author, 'synthetic': True, 'metadata': metadata, 'parents': parents, } def build_gitsvn_swh_revision(rev, commit, dir_id, parents): """Given a svn revision, build a swh revision. Args: - rev: the svn revision number - commit: the commit metadata - dir_id: the tree's hash identifier - parents: the revision's parents identifier Returns: The swh revision dictionary. """ author = commit['author_name'] msg = commit['message'] date = commit['author_date'] return { 'date': date, 'committer_date': date, 'type': 'svn', 'directory': dir_id, 'message': msg, 'author': author, 'committer': author, 'synthetic': True, 'metadata': None, 'parents': parents, } def build_swh_occurrence(revision_id, origin_id, visit): """Build a swh occurrence from the revision id, origin id, and date. """ return {'branch': 'master', 'target': revision_id, 'target_type': 'revision', 'origin': origin_id, 'visit': visit} - - -def loader_to_scheduler_revision(swh_revision): - """To avoid serialization or scheduler storage problem, transform - adequately the revision. - - FIXME: Should be more generically dealt with in swh-scheduler's - side. The advantage to having it here is that we known what we - store. - - """ - if not swh_revision: - return None - - metadata = swh_revision['metadata'] - for entry in (e for e in metadata['extra_headers'] - if isinstance(e[1], bytes)): - entry[1] = entry[1].decode('utf-8') - - return { - 'id': hashutil.hash_to_hex(swh_revision['id']), - 'parents': [hashutil.hash_to_hex(parent) for parent - in swh_revision['parents']], - 'metadata': metadata - } - - -def scheduler_to_loader_revision(swh_revision): - """If the known state (a revision) is already passed, it will be - serializable ready but not loader ready. - - FIXME: Should be more generically dealt with in swh-scheduler's - side. The advantage to having it here is that we known what we - store. - - """ - if not swh_revision: - return None - return { - 'id': hashutil.hex_to_hash(swh_revision['id']), - 'parents': [hashutil.hex_to_hash(parent) for parent - in swh_revision['parents']], - 'metadata': swh_revision['metadata'] - } diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py index 5ed2444..5745c4c 100644 --- a/swh/loader/svn/loader.py +++ b/swh/loader/svn/loader.py @@ -1,502 +1,502 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Loader in charge of injecting either new or existing svn mirrors to swh-storage. """ import abc import os import shutil -from swh.core import utils, hashutil -from swh.model import git +from swh.core import utils +from swh.model import git, hashutil from swh.model.git import GitType from swh.loader.core.loader import SWHLoader from . import svn, converters from .utils import hashtree, init_svn_repo_from_archive_dump class SvnLoaderEventful(ValueError): """A wrapper exception to transit the swh_revision onto which the loading failed. """ def __init__(self, e, swh_revision): super().__init__(e) self.swh_revision = swh_revision class SvnLoaderUneventful(ValueError): pass class SvnLoaderHistoryAltered(ValueError): pass class BaseSvnLoader(SWHLoader, metaclass=abc.ABCMeta): """Base Svn loader to load one svn repository according to specific policies (only swh one now). The main entry point of this is (no need to override it) - def load(self, origin_visit, last_known_swh_revision=None): pass Inherit this class and then override the following functions: - def build_swh_revision(self, rev, commit, dir_id, parents) This is in charge of converting an svn revision to a compliant swh revision - def process_repository(self) This is in charge of processing the actual svn repository and store the result to swh storage. """ CONFIG_BASE_FILENAME = 'loader/svn' ADDITIONAL_CONFIG = { 'check_revision': ('int', 1000), } def __init__(self): super().__init__(logging_class='swh.loader.svn.SvnLoader') self.check_revision = self.config['check_revision'] @abc.abstractmethod def swh_revision_hash_tree_at_svn_revision(self, revision): """Compute and return the hash tree at a given svn revision. Args: rev (int): the svn revision we want to check Returns: The hash tree directory as bytes. """ pass @abc.abstractmethod def get_svn_repo(self, svn_url, destination_path, origin): """Instantiates the needed svnrepo collaborator to permit reading svn repository. Args: svn_url: the svn repository url to read from destination_path: the local path on disk to compute data origin: the corresponding origin Returns: Instance of swh.loader.svn.svn clients """ raise NotImplementedError @abc.abstractmethod def build_swh_revision(self, rev, commit, dir_id, parents): """Convert an svn revision to an swh one according to the loader's policy (git-svn or swh). Args: rev: the svn revision number commit: dictionary with keys 'author_name', 'author_date', 'rev', 'message' dir_id: the hash tree computation parents: the revision's parents Returns: The swh revision """ raise NotImplementedError @abc.abstractmethod def process_repository(self, origin_visit, last_known_swh_revision=None): """The main idea of this function is to: - iterate over the svn commit logs - extract the svn commit log metadata - compute the hashes from the current directory down to the file - compute the equivalent swh revision - send all those objects for storage - create an swh occurrence pointing to the last swh revision seen - send that occurrence for storage in swh-storage. """ raise NotImplementedError def process_svn_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process revisions from revision_start to revision_end and send to swh for storage. At each svn revision, checkout the repository, compute the tree hash and blobs and send for swh storage to store. Then computes and yields the swh revision. Note that at every self.check_revision, an svn export is done and a hash tree is computed to check that no divergence occurred. Yields: swh revision as a dictionary with keys, sha1_git, sha1, etc... """ gen_revs = svnrepo.swh_hash_data_per_revision( revision_start, revision_end) swh_revision = None count = 0 for rev, nextrev, commit, objects_per_path in gen_revs: count += 1 # Send the associated contents/directories self.maybe_load_contents( git.objects_per_type(GitType.BLOB, objects_per_path)) self.maybe_load_directories( git.objects_per_type(GitType.TREE, objects_per_path)) # compute the fs tree's checksums dir_id = objects_per_path[b'']['checksums']['sha1_git'] swh_revision = self.build_swh_revision( rev, commit, dir_id, revision_parents[rev]) swh_revision['id'] = git.compute_revision_sha1_git( swh_revision) self.log.debug('rev: %s, swhrev: %s, dir: %s' % ( rev, hashutil.hash_to_hex(swh_revision['id']), hashutil.hash_to_hex(dir_id))) if (count % self.check_revision) == 0: # hash computation check self.log.info('Checking hash computations on revision %s...' % rev) checked_dir_id = self.swh_revision_hash_tree_at_svn_revision( rev) if checked_dir_id != dir_id: err = 'Hash tree computation divergence detected (%s != %s), stopping!' % ( # noqa hashutil.hash_to_hex(dir_id), hashutil.hash_to_hex(checked_dir_id) ) raise ValueError(err) if nextrev: revision_parents[nextrev] = [swh_revision['id']] yield swh_revision def process_swh_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process and store revision to swh (sent by blocks of 'revision_packet_size') Returns: The latest revision stored. """ try: swh_revision_gen = self.process_svn_revisions(svnrepo, revision_start, revision_end, revision_parents) revs = [] for revisions in utils.grouper( swh_revision_gen, self.config['revision_packet_size']): revs = list(revisions) self.log.info('Processed %s revisions: [%s, ...]' % ( len(revs), hashutil.hash_to_hex(revs[0]['id']))) self.maybe_load_revisions(revs) except Exception as e: if revs: # flush remaining revisions self.maybe_load_revisions(revs) # Take the last one as the last known revisions known_swh_rev = revs[-1] # Then notify something is wrong, and we stopped at that rev. raise SvnLoaderEventful(e, swh_revision={ 'id': known_swh_rev['id'], }) raise e return revs[-1] def process_swh_occurrence(self, revision, origin_visit): """Process and load the occurrence pointing to the latest revision. """ occ = converters.build_swh_occurrence( revision['id'], origin_visit['origin'], origin_visit['visit']) self.log.debug('occ: %s' % occ) self.maybe_load_occurrences([occ]) def prepare(self, *args, **kwargs): destination_path = kwargs['destination_path'] # local svn url svn_url = kwargs['svn_url'] origin_url = kwargs.get('origin_url') self.visit_date = kwargs.get('visit_date') origin = { 'url': origin_url if origin_url else svn_url, 'type': 'svn', } self.origin_id = self.send_origin(origin) origin['id'] = self.origin_id self.origin = origin if 'swh_revision' in kwargs: - self.last_known_swh_revision = hashutil.hex_to_hash( + self.last_known_swh_revision = hashutil.hash_to_bytes( kwargs['swh_revision']) else: self.last_known_swh_revision = None self.svnrepo = self.get_svn_repo(svn_url, destination_path, origin) self.fetch_history_id = self.open_fetch_history() def get_origin(self): """Retrieve the origin we are working with. """ return self.origin # set in prepare method def fetch_data(self): """We need to fetch and stream the data to store directly. So fetch_data do actually nothing. The method `store_data` below is in charge to do everything, fetch and store. """ pass def store_data(self): """We need to fetch and stream the data to store directly because there is too much data and state changes. Everything is intertwined together (We receive patch and apply on disk and compute at the hashes at the same time) So every data to fetch and store is done here. Requisite: origin_visit and last_known_swh_revision must have been set in the prepare method. """ origin_visit = {'origin': self.origin_id, 'visit': self.visit} try: latest_rev = self.process_repository(origin_visit, self.last_known_swh_revision) except SvnLoaderEventful as e: self.log.error('Eventful partial visit. Detail: %s' % e) latest_rev = e.swh_revision self.process_swh_occurrence(latest_rev, origin_visit) raise except (SvnLoaderHistoryAltered, SvnLoaderUneventful) as e: self.log.error('Uneventful visit. Detail: %s' % e) raise except Exception as e: raise else: self.process_swh_occurrence(latest_rev, origin_visit) class SWHSvnLoader(BaseSvnLoader): """Swh svn loader is the main implementation destined for production. This implementation is able to deal with update on known svn repository. Default policy: Keep data as close as possible from the original svn data. We only add information that are needed for update or continuing from last known revision (svn revision and svn repository's uuid). """ def __init__(self): super().__init__() def cleanup(self): """Clean after oneself. This is in charge to flush the remaining data to write in swh storage. And to clean up the svn repository's working representation on disk. """ self.svnrepo.clean_fs() def swh_revision_hash_tree_at_svn_revision(self, revision): """Compute a given hash tree at specific revision. """ local_dirname, local_url = self.svnrepo.export_temporary(revision) h = hashtree(local_url)['sha1_git'] self.svnrepo.clean_fs(local_dirname) return h def get_svn_repo(self, svn_url, destination_path, origin): return svn.SWHSvnRepo( svn_url, origin['id'], self.storage, destination_path=destination_path) def swh_previous_revision(self, prev_swh_revision=None): """Retrieve swh's previous revision if any. """ return self.svnrepo.swh_previous_revision(prev_swh_revision) def check_history_not_altered(self, svnrepo, revision_start, swh_rev): """Given a svn repository, check if the history was not tampered with. """ revision_id = swh_rev['id'] parents = swh_rev['parents'] hash_data_per_revs = svnrepo.swh_hash_data_at_revision(revision_start) rev = revision_start rev, _, commit, objects_per_path = list(hash_data_per_revs)[0] dir_id = objects_per_path[b'']['checksums']['sha1_git'] swh_revision = self.build_swh_revision(rev, commit, dir_id, parents) swh_revision_id = git.compute_revision_sha1_git(swh_revision) return swh_revision_id == revision_id def build_swh_revision(self, rev, commit, dir_id, parents): """Build the swh revision dictionary. This adds: - the 'synthetic' flag to true - the 'extra_headers' containing the repository's uuid and the svn revision number. Args: rev: the svn revision commit: the commit metadata dir_id: the upper tree's hash identifier parents: the parents' identifiers Returns: The swh revision corresponding to the svn revision. """ return converters.build_swh_revision(rev, commit, self.svnrepo.uuid, dir_id, parents) def init_from(self, partial_swh_revision, previous_swh_revision): """Function to determine from where to start from. Args: - partial_swh_revision: A known revision from which the previous loading did not finish. - known_previous_revision: A known revision from which the previous loading did finish. Returns: The revision from which to start or None if nothing (fresh start). """ if partial_swh_revision and not previous_swh_revision: return partial_swh_revision if not partial_swh_revision and previous_swh_revision: return previous_swh_revision if partial_swh_revision and previous_swh_revision: # will determine from which to start from extra_headers1 = dict( partial_swh_revision['metadata']['extra_headers']) extra_headers2 = dict( previous_swh_revision['metadata']['extra_headers']) rev_start1 = int(extra_headers1['svn_revision']) rev_start2 = int(extra_headers2['svn_revision']) if rev_start1 <= rev_start2: return previous_swh_revision return partial_swh_revision return None def process_repository(self, origin_visit, last_known_swh_revision=None): svnrepo = self.svnrepo # default configuration revision_start = 1 revision_parents = { revision_start: [] } # Check if we already know a previous revision for that origin swh_rev = self.swh_previous_revision() # Determine from which known revision to start swh_rev = self.init_from(last_known_swh_revision, previous_swh_revision=swh_rev) if swh_rev: # Yes, we do. Try and update it. extra_headers = dict(swh_rev['metadata']['extra_headers']) revision_start = int(extra_headers['svn_revision']) revision_parents = { revision_start: swh_rev['parents'], } self.log.debug('svn export --ignore-keywords %s@%s' % ( svnrepo.remote_url, revision_start)) if swh_rev and not self.check_history_not_altered( svnrepo, revision_start, swh_rev): msg = 'History of svn %s@%s history modified. Skipping...' % ( svnrepo.remote_url, revision_start) self.log.warn(msg) raise SvnLoaderHistoryAltered else: # now we know history is ok, we start at next revision revision_start = revision_start + 1 # and the parent become the latest know revision for # that repository revision_parents[revision_start] = [swh_rev['id']] revision_end = svnrepo.head_revision() self.log.info('[revision_start-revision_end]: [%s-%s]' % ( revision_start, revision_end)) if revision_start > revision_end and revision_start is not 1: self.log.info('%s@%s already injected.' % ( svnrepo.remote_url, revision_end)) raise SvnLoaderUneventful self.log.info('Processing %s.' % svnrepo) # process and store revision to swh (sent by by blocks of # 'revision_packet_size') return self.process_swh_revisions( svnrepo, revision_start, revision_end, revision_parents) class SWHSvnLoaderFromDumpArchive(SWHSvnLoader): """Load a svn repository from an archive (containing a dump). """ def __init__(self, archive_path): super().__init__() self.log.info('Archive to mount and load %s' % archive_path) self.temp_dir, self.repo_path = init_svn_repo_from_archive_dump( archive_path) def cleanup(self): super().cleanup() if self.temp_dir and os.path.exists(self.temp_dir): self.log.debug('Clean up temp directory %s for project %s' % ( self.temp_dir, os.path.basename(self.repo_path))) shutil.rmtree(self.temp_dir) diff --git a/swh/loader/svn/tests/test_converters.py b/swh/loader/svn/tests/test_converters.py index 6a90bcc..ec1d201 100644 --- a/swh/loader/svn/tests/test_converters.py +++ b/swh/loader/svn/tests/test_converters.py @@ -1,368 +1,310 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest from nose.tools import istest from swh.loader.svn import converters class TestAuthorGitSvnConverters(unittest.TestCase): @istest def svn_author_to_gitsvn_person(self): """The author should have name, email and fullname filled. """ actual_person = converters.svn_author_to_gitsvn_person( 'tony ', repo_uuid=None) self.assertEquals(actual_person, { 'fullname': b'tony ', 'name': b'tony', 'email': b'ynot@dagobah', }) @istest def svn_author_to_gitsvn_person_no_email(self): """The author should see his/her email filled with author@. """ actual_person = converters.svn_author_to_gitsvn_person( 'tony', repo_uuid=b'some-uuid') self.assertEquals(actual_person, { 'fullname': b'tony ', 'name': b'tony', 'email': b'tony@some-uuid', }) @istest def svn_author_to_gitsvn_person_empty_person(self): """The empty person should see name, fullname and email filled. """ actual_person = converters.svn_author_to_gitsvn_person( '', repo_uuid=b'some-uuid') self.assertEqual(actual_person, { 'fullname': b'(no author) <(no author)@some-uuid>', 'name': b'(no author)', 'email': b'(no author)@some-uuid' }) class TestAuthorSWHConverters(unittest.TestCase): @istest def svn_author_to_swh_person(self): """The author should have name, email and fullname filled. """ actual_person = converters.svn_author_to_swh_person( 'tony ') self.assertEquals(actual_person, { 'fullname': b'tony ', 'name': b'tony', 'email': b'ynot@dagobah', }) @istest def svn_author_to_swh_person_no_email(self): """The author and fullname should be the same as the input (author). """ actual_person = converters.svn_author_to_swh_person('tony') self.assertEquals(actual_person, { 'fullname': b'tony', 'name': b'tony', 'email': None, }) @istest def svn_author_to_swh_person_empty_person(self): """Empty person has only its fullname filled with the empty byte-string. """ actual_person = converters.svn_author_to_swh_person('') self.assertEqual(actual_person, { 'fullname': b'', 'name': None, 'email': None, }) class TestSWHRevisionConverters(unittest.TestCase): @istest def build_swh_revision_default(self): """This should build the swh revision with the swh revision's extra headers about the repository. """ actual_swh_revision = converters.build_swh_revision( repo_uuid=b'uuid', dir_id='dir-id', commit={ 'author_name': { 'name': b'theo', 'email': b'theo@uuid', 'fullname': b'theo ' }, 'message': b'commit message', 'author_date': { 'timestamp': { 'seconds': 1088108379, 'microseconds': 0, }, 'offset': 0 } }, rev=10, parents=['123']) date = { 'timestamp': { 'seconds': 1088108379, 'microseconds': 0, }, 'offset': 0, } self.assertEquals(actual_swh_revision, { 'date': date, 'committer_date': date, 'type': 'svn', 'directory': 'dir-id', 'message': b'commit message', 'author': { 'name': b'theo', 'email': b'theo@uuid', 'fullname': b'theo ' }, 'committer': { 'name': b'theo', 'email': b'theo@uuid', 'fullname': b'theo ' }, 'synthetic': True, 'metadata': { 'extra_headers': [ ['svn_repo_uuid', b'uuid'], ['svn_revision', b'10'], ] }, 'parents': ['123'], }) class TestGitSvnRevisionConverters(unittest.TestCase): @istest def build_gitsvn_swh_revision_default(self): """This should build the swh revision without the swh revision's extra headers about the repository. """ actual_swh_revision = converters.build_gitsvn_swh_revision( dir_id='dir-id', commit={ 'author_name': { 'name': b'theo', 'email': b'theo@uuid', 'fullname': b'theo ' }, 'message': b'commit message', 'author_date': { 'timestamp': { 'seconds': 1088108379, 'microseconds': 0, }, 'offset': 0 } }, rev=10, parents=['123']) date = { 'timestamp': { 'seconds': 1088108379, 'microseconds': 0, }, 'offset': 0, } self.assertEquals(actual_swh_revision, { 'date': date, 'committer_date': date, 'type': 'svn', 'directory': 'dir-id', 'message': b'commit message', 'author': { 'name': b'theo', 'email': b'theo@uuid', 'fullname': b'theo ' }, 'committer': { 'name': b'theo', 'email': b'theo@uuid', 'fullname': b'theo ' }, 'synthetic': True, 'metadata': None, 'parents': ['123'], }) class TestSWHOccurrence(unittest.TestCase): @istest def build_swh_occurrence(self): actual_occ = converters.build_swh_occurrence('revision-id', 'origin-id', visit=10) self.assertEquals(actual_occ, { 'branch': 'master', 'target': 'revision-id', 'target_type': 'revision', 'origin': 'origin-id', 'visit': 10 }) class ConvertSWHDate(unittest.TestCase): @istest def svn_date_to_swh_date(self): """The timestamp should not be tampered with and include the decimals. """ self.assertEquals( converters.svn_date_to_swh_date('2011-05-31T06:04:39.500900Z'), { 'timestamp': { 'seconds': 1306821879, 'microseconds': 500900, }, 'offset': 0 }) self.assertEquals( converters.svn_date_to_swh_date('2011-05-31T06:04:39.800722Z'), { 'timestamp': { 'seconds': 1306821879, 'microseconds': 800722, }, 'offset': 0 }) @istest def svn_date_to_swh_date_epoch(self): """Empty date should be EPOCH (timestamp and offset at 0).""" # It should return 0, epoch self.assertEquals({ 'timestamp': { 'seconds': 0, 'microseconds': 0, }, 'offset': 0, }, converters.svn_date_to_swh_date('')) self.assertEquals({ 'timestamp': { 'seconds': 0, 'microseconds': 0, }, 'offset': 0, }, converters.svn_date_to_swh_date(None)) class ConvertGitSvnDate(unittest.TestCase): @istest def svn_date_to_gitsvn_date(self): """The timestamp should be truncated to be an integer.""" actual_ts = converters.svn_date_to_gitsvn_date( '2011-05-31T06:04:39.800722Z') self.assertEquals(actual_ts, { 'timestamp': { 'seconds': 1306821879, 'microseconds': 0, }, 'offset': 0, }) @istest def svn_date_to_gitsvn_date_epoch(self): """Empty date should be EPOCH (timestamp and offset at 0).""" # It should return 0, epoch self.assertEquals({ 'timestamp': { 'seconds': 0, 'microseconds': 0, }, 'offset': 0, }, converters.svn_date_to_gitsvn_date('')) self.assertEquals({ 'timestamp': { 'seconds': 0, 'microseconds': 0, }, 'offset': 0, }, converters.svn_date_to_gitsvn_date(None)) - - -class ConvertSWHRevision(unittest.TestCase): - @istest - def loader_to_scheduler_revision(self): - actual_rev = converters.loader_to_scheduler_revision({ - 'parents': [b'e\n\xbe\xe9\xc0\x87y\xfeG\xf7\xcfG\x82h\xa8i\xe8\xfe\xe2\x13'], # noqa - 'id': b'\xedd\x92w\xab\xb2\x16,\xea*\x90O8\x0f\x96/\xfb\xd4\x16`', - 'metadata': { - 'extra_headers': [ - ['svn_repo_uuid', b'bc7d6c17-68a5-4917-9c54-c565d7424229'], - ['svn_revision', b'4'] - ] - } - }) - - self.assertEquals(actual_rev, { - 'id': 'ed649277abb2162cea2a904f380f962ffbd41660', - 'parents': ['650abee9c08779fe47f7cf478268a869e8fee213'], - 'metadata': { - 'extra_headers': [ - ['svn_repo_uuid', 'bc7d6c17-68a5-4917-9c54-c565d7424229'], - ['svn_revision', '4'] - ] - } - }) - - @istest - def loader_to_scheduler_revision_none(self): - self.assertIsNone(converters.loader_to_scheduler_revision(None)) - - @istest - def scheduler_to_loader_revision(self): - actual_rev = converters.scheduler_to_loader_revision({ - 'id': 'ed649277abb2162cea2a904f380f962ffbd41660', - 'parents': ['650abee9c08779fe47f7cf478268a869e8fee213'], - 'metadata': { - 'extra_headers': [ - ['svn_repo_uuid', 'bc7d6c17-68a5-4917-9c54-c565d7424229'], - ['svn_revision', '4'] - ] - } - }) - - self.assertEquals(actual_rev, { - 'parents': [b'e\n\xbe\xe9\xc0\x87y\xfeG\xf7\xcfG\x82h\xa8i\xe8\xfe\xe2\x13'], # noqa - 'id': b'\xedd\x92w\xab\xb2\x16,\xea*\x90O8\x0f\x96/\xfb\xd4\x16`', - 'metadata': { - 'extra_headers': [ - ['svn_repo_uuid', 'bc7d6c17-68a5-4917-9c54-c565d7424229'], - ['svn_revision', '4'] - ] - } - }) - - @istest - def scheduler_to_loader_revision_none(self): - self.assertIsNone(converters.scheduler_to_loader_revision(None)) diff --git a/swh/loader/svn/tests/test_loader.py b/swh/loader/svn/tests/test_loader.py index 745f126..b0d0e04 100644 --- a/swh/loader/svn/tests/test_loader.py +++ b/swh/loader/svn/tests/test_loader.py @@ -1,619 +1,619 @@ -# Copyright (C) 2016 The Software Heritage developers +# Copyright (C) 2016-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from nose.tools import istest -from swh.core import hashutil +from swh.model import hashutil from swh.loader.svn.loader import SWHSvnLoader from swh.loader.svn.loader import SvnLoaderHistoryAltered, SvnLoaderUneventful from test_base import BaseTestSvnLoader # Define loaders with no storage # They'll just accumulate the data in place # Only for testing purposes. class TestSvnLoader: """Mixin class to inhibit the persistence and keep in memory the data sent for storage. cf. SWHSvnLoaderNoStorage """ def __init__(self): super().__init__() # We don't want to persist any result in this test context self.config['send_contents'] = False self.config['send_directories'] = False self.config['send_revisions'] = False self.config['send_releases'] = False self.config['send_occurrences'] = False # Init the state self.all_contents = [] self.all_directories = [] self.all_revisions = [] self.all_releases = [] self.all_occurrences = [] # Check at each svn revision that the hash tree computation # does not diverge self.check_revision = 1 def maybe_load_contents(self, all_contents): self.all_contents.extend(all_contents) def maybe_load_directories(self, all_directories): self.all_directories.extend(all_directories) def maybe_load_revisions(self, all_revisions): self.all_revisions.extend(all_revisions) def maybe_load_releases(self, releases): raise ValueError('If called, the test must break.') def maybe_load_occurrences(self, all_occurrences): self.all_occurrences.extend(all_occurrences) # Override to do nothing at the end def close_failure(self): pass def close_success(self): pass # Override to only prepare the svn repository def prepare(self, *args, **kwargs): self.svnrepo = self.get_svn_repo(*args) class SWHSvnLoaderNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: Load a new svn repository using the swh policy (so no update). """ def swh_previous_revision(self, prev_swh_revision=None): """We do not know this repository so no revision. """ return None class SWHSvnLoaderUpdateNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: Load a known svn repository using the swh policy. We can either: - do nothing since it does not contain any new commit (so no change) - either check its history is not altered and update in consequence by loading the new revision """ def swh_previous_revision(self, prev_swh_revision=None): """Avoid the storage persistence call and return the expected previous revision for that repository. Check the following for explanation about the hashes: - test_loader.org for (swh policy). - cf. SWHSvnLoaderITTest """ return { - 'id': hashutil.hex_to_hash( + 'id': hashutil.hash_to_bytes( '4876cb10aec6f708f7466dddf547567b65f6c39c'), - 'parents': [hashutil.hex_to_hash( + 'parents': [hashutil.hash_to_bytes( 'a3a577948fdbda9d1061913b77a1588695eadb41')], - 'directory': hashutil.hex_to_hash( + 'directory': hashutil.hash_to_bytes( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '6'] ] } } class SWHSvnLoaderUpdateHistoryAlteredNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: Load a known svn repository using the swh policy with its history altered so we do not update it. """ def swh_previous_revision(self, prev_swh_revision=None): """Avoid the storage persistence call and return the expected previous revision for that repository. Check the following for explanation about the hashes: - test_loader.org for (swh policy). - cf. SWHSvnLoaderITTest """ return { # Changed the revision id's hash to simulate history altered - 'id': hashutil.hex_to_hash( + 'id': hashutil.hash_to_bytes( 'badbadbadbadf708f7466dddf547567b65f6c39d'), - 'parents': [hashutil.hex_to_hash( + 'parents': [hashutil.hash_to_bytes( 'a3a577948fdbda9d1061913b77a1588695eadb41')], - 'directory': hashutil.hex_to_hash( + 'directory': hashutil.hash_to_bytes( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', b'6'] ] } } class SWHSvnLoaderNewRepositoryITTest(BaseTestSvnLoader): def setUp(self): super().setUp() self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.origin_visit = { 'origin': self.origin['id'], 'visit': 2, } self.loader = SWHSvnLoaderNoStorage() self.loader.prepare( self.svn_mirror_url, self.destination_path, self.origin) @istest def process_repository(self): """Process a new repository with swh policy should be ok. """ # when self.loader.process_repository(self.origin_visit) # then self.assertEquals(len(self.loader.all_revisions), 6) self.assertEquals(len(self.loader.all_releases), 0) last_revision = '4876cb10aec6f708f7466dddf547567b65f6c39c' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71': '669a71cce6c424a81ba42b7dc5d560d32252f0ca', # noqa '95edacc8848369d6fb1608e887d6d2474fd5224f': '008ac97a1118560797c50e3392fa1443acdaa349', # noqa 'fef26ea45a520071711ba2b9d16a2985ee837021': '3780effbe846a26751a95a8c95c511fb72be15b4', # noqa '3f51abf3b3d466571be0855dfa67e094f9ceff1b': 'ffcca9b09c5827a6b8137322d4339c8055c3ee1e', # noqa 'a3a577948fdbda9d1061913b77a1588695eadb41': '7dc52cc04c3b8bd7c085900d60c159f7b846f866', # noqa last_revision: '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) class SWHSvnLoaderUpdateWithNoChangeITTest(BaseTestSvnLoader): def setUp(self): super().setUp() self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.origin_visit = { 'origin': self.origin['id'], 'visit': 3, } self.loader = SWHSvnLoaderUpdateNoStorage() self.loader.prepare( self.svn_mirror_url, self.destination_path, self.origin) @istest def process_repository(self): """Process a known repository with swh policy and no new data should be ok. """ # when with self.assertRaises(SvnLoaderUneventful): self.loader.process_repository(self.origin_visit) # then self.assertEquals(len(self.loader.all_revisions), 0) self.assertEquals(len(self.loader.all_releases), 0) class SWHSvnLoaderUpdateWithHistoryAlteredITTest(BaseTestSvnLoader): def setUp(self): # the svn repository pkg-gourmet has been updated with changes super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.origin_visit = { 'origin': self.origin['id'], 'visit': 4, } self.loader = SWHSvnLoaderUpdateHistoryAlteredNoStorage() self.loader.prepare( self.svn_mirror_url, self.destination_path, self.origin) @istest def process_repository(self): """Process a known repository with swh policy and history altered should stop and do nothing. """ # when with self.assertRaises(SvnLoaderHistoryAltered): self.loader.process_repository(self.origin_visit) # then # we got the previous run's last revision (rev 6) # so 2 news + 1 old self.assertEquals(len(self.loader.all_revisions), 0) self.assertEquals(len(self.loader.all_releases), 0) class SWHSvnLoaderUpdateWithChangesITTest(BaseTestSvnLoader): def setUp(self): # the svn repository pkg-gourmet has been updated with changes super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.origin_visit = { 'origin': self.origin['id'], 'visit': 5, } self.loader = SWHSvnLoaderUpdateNoStorage() self.loader.prepare( self.svn_mirror_url, self.destination_path, self.origin) @istest def process_repository(self): """Process a known repository with swh policy and new data should yield new revisions and occurrence. """ # when self.loader.process_repository(self.origin_visit) # then # we got the previous run's last revision (rev 6) # so 2 new self.assertEquals(len(self.loader.all_revisions), 5) self.assertEquals(len(self.loader.all_releases), 0) last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) class SWHSvnLoaderUpdateWithUnfinishedLoadingChangesITTest(BaseTestSvnLoader): def setUp(self): super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.origin_visit = { 'origin': self.origin['id'], 'visit': 6 } self.loader = SWHSvnLoaderNoStorage() self.loader.prepare( self.svn_mirror_url, self.destination_path, self.origin) @istest def process_repository(self): """Process a known repository with swh policy, the previous run did not finish, so this finishes the loading """ previous_unfinished_revision = { - 'id': hashutil.hex_to_hash( + 'id': hashutil.hash_to_bytes( '4876cb10aec6f708f7466dddf547567b65f6c39c'), - 'parents': [hashutil.hex_to_hash( + 'parents': [hashutil.hash_to_bytes( 'a3a577948fdbda9d1061913b77a1588695eadb41')], - 'directory': hashutil.hex_to_hash( + 'directory': hashutil.hash_to_bytes( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '6'] ] } } # when self.loader.process_repository( self.origin_visit, last_known_swh_revision=previous_unfinished_revision) # then # we got the previous run's last revision (rev 6) # so 2 new self.assertEquals(len(self.loader.all_revisions), 5) self.assertEquals(len(self.loader.all_releases), 0) last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) class SWHSvnLoaderUpdateWithUnfinishedLoadingChangesButOccurrenceDoneITTest( BaseTestSvnLoader): def setUp(self): super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.origin_visit = { 'origin': self.origin['id'], 'visit': 9, } self.loader = SWHSvnLoaderUpdateNoStorage() self.loader.prepare( self.svn_mirror_url, self.destination_path, self.origin) @istest def process_repository(self): """known repository, swh policy, unfinished revision is less recent than occurrence, we start from last occurrence. """ previous_unfinished_revision = { - 'id': hashutil.hex_to_hash( + 'id': hashutil.hash_to_bytes( 'a3a577948fdbda9d1061913b77a1588695eadb41'), - 'parents': [hashutil.hex_to_hash( + 'parents': [hashutil.hash_to_bytes( '3f51abf3b3d466571be0855dfa67e094f9ceff1b')], - 'directory': hashutil.hex_to_hash( + 'directory': hashutil.hash_to_bytes( '7dc52cc04c3b8bd7c085900d60c159f7b846f866'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '5'] ] } } # when self.loader.process_repository( self.origin_visit, last_known_swh_revision=previous_unfinished_revision) # then # we got the previous run's last revision (rev 6) # so 2 new self.assertEquals(len(self.loader.all_revisions), 5) self.assertEquals(len(self.loader.all_releases), 0) last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) class SWHSvnLoaderUpdateLessRecentNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: Load a known svn repository using the swh policy. The last occurrence seen is less recent than a previous unfinished crawl. """ def swh_previous_revision(self, prev_swh_revision=None): """Avoid the storage persistence call and return the expected previous revision for that repository. Check the following for explanation about the hashes: - test_loader.org for (swh policy). - cf. SWHSvnLoaderITTest """ return { - 'id': hashutil.hex_to_hash( + 'id': hashutil.hash_to_bytes( 'a3a577948fdbda9d1061913b77a1588695eadb41'), - 'parents': [hashutil.hex_to_hash( + 'parents': [hashutil.hash_to_bytes( '3f51abf3b3d466571be0855dfa67e094f9ceff1b')], - 'directory': hashutil.hex_to_hash( + 'directory': hashutil.hash_to_bytes( '7dc52cc04c3b8bd7c085900d60c159f7b846f866'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '5'] ] } } class SWHSvnLoaderUnfinishedLoadingChangesSinceLastOccurrenceITTest( BaseTestSvnLoader): def setUp(self): super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.origin_visit = { 'origin': self.origin['id'], 'visit': 1, } self.loader = SWHSvnLoaderUpdateLessRecentNoStorage() self.loader.prepare( self.svn_mirror_url, self.destination_path, self.origin) @istest def process_repository(self): """known repository, swh policy, unfinished revision is less recent than occurrence, we start from last occurrence. """ previous_unfinished_revision = { - 'id': hashutil.hex_to_hash( + 'id': hashutil.hash_to_bytes( '4876cb10aec6f708f7466dddf547567b65f6c39c'), - 'parents': [hashutil.hex_to_hash( + 'parents': [hashutil.hash_to_bytes( 'a3a577948fdbda9d1061913b77a1588695eadb41')], - 'directory': hashutil.hex_to_hash( + 'directory': hashutil.hash_to_bytes( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '6'] ] } } # when self.loader.process_repository( self.origin_visit, last_known_swh_revision=previous_unfinished_revision) # then # we got the previous run's last revision (rev 6) # so 2 new self.assertEquals(len(self.loader.all_revisions), 5) self.assertEquals(len(self.loader.all_releases), 0) last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) class SWHSvnLoaderUpdateAndTestCornerCasesAboutEolITTest(BaseTestSvnLoader): def setUp(self): super().setUp(archive_name='pkg-gourmet-with-eol-corner-cases.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.origin_visit = { 'origin': self.origin['id'], 'visit': 1, } self.loader = SWHSvnLoaderUpdateLessRecentNoStorage() self.loader.prepare( self.svn_mirror_url, self.destination_path, self.origin) @istest def process_repository(self): """EOL corner cases and update. """ previous_unfinished_revision = { - 'id': hashutil.hex_to_hash( + 'id': hashutil.hash_to_bytes( '171dc35522bfd17dda4e90a542a0377fb2fc707a'), - 'parents': [hashutil.hex_to_hash( + 'parents': [hashutil.hash_to_bytes( '902f29b4323a9b9de3af6d28e72dd581e76d9397')], - 'directory': hashutil.hex_to_hash( + 'directory': hashutil.hash_to_bytes( 'fd24a76c87a3207428e06612b49860fc78e9f6dc'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '11'] ] } } # when self.loader.process_repository( self.origin_visit, last_known_swh_revision=previous_unfinished_revision) # then # we got the previous run's last revision (rev 11) # so 8 new self.assertEquals(len(self.loader.all_revisions), 8) self.assertEquals(len(self.loader.all_releases), 0) last_revision = '0148ae3eaa520b73a50802c59f3f416b7a36cf8c' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '027e8769f4786597436ab94a91f85527d04a6cbb': '2d9ca72c6afec6284fb01e459588cbb007017c8c', # noqa '4474d96018877742d9697d5c76666c9693353bfc': 'ab111577e0ab39e4a157c476072af48f2641d93f', # noqa '97ad21eab92961e2a22ca0285f09c6d1e9a7ffbc': 'ab111577e0ab39e4a157c476072af48f2641d93f', # noqa 'd04ea8afcee6205cc8384c091bfc578931c169fd': 'b0a648b02e55a4dce356ac35187a058f89694ec7', # noqa 'ded78810401fd354ffe894aa4a1e5c7d30a645d1': 'b0a648b02e55a4dce356ac35187a058f89694ec7', # noqa '4ee95e39358712f53c4fc720da3fafee9249ed19': 'c3c98df624733fef4e592bef983f93e2ed02b179', # noqa 'ffa901b69ca0f46a2261f42948838d19709cb9f8': 'c3c98df624733fef4e592bef983f93e2ed02b179', # noqa last_revision: '844d4646d6c2b4f3a3b2b22ab0ee38c7df07bab2', # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) diff --git a/swh/loader/svn/utils.py b/swh/loader/svn/utils.py index e82db15..9a35fff 100644 --- a/swh/loader/svn/utils.py +++ b/swh/loader/svn/utils.py @@ -1,178 +1,178 @@ # Copyright (C) 2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import tempfile import shutil from dateutil import parser from subprocess import PIPE, Popen, call from swh.model import git def strdate_to_timestamp(strdate): """Convert a string date to an int timestamp. Args: strdate: A string representing a date with format like 'YYYY-mm-DDTHH:MM:SS.800722Z' Returns: A couple of integers: seconds, microseconds """ if strdate: dt = parser.parse(strdate) ts = { 'seconds': int(dt.timestamp()), 'microseconds': dt.microsecond, } else: # epoch ts = {'seconds': 0, 'microseconds': 0} return ts def convert_hashes_with_relative_path(hashes, rootpath): """A function to ease the transformation of absolute path to relative ones. This is an implementation detail: - swh.loader.svn.ra compute hashes and store keys with relative paths - swh.model.git compute hashes and store keys with full paths """ if rootpath.endswith(b'/'): rootpath = rootpath[:-1] root_value = hashes.pop(rootpath) if not rootpath.endswith(b'/'): rootpath = rootpath + b'/' def _replace_slash(s, rootpath=rootpath): return s.replace(rootpath, b'') def _update_children(children): return set((_replace_slash(c) for c in children)) h = { b'': { 'checksums': root_value['checksums'], 'children': _update_children(root_value['children']) } } for path, v in hashes.items(): p = _replace_slash(path) if 'children' in v: v['children'] = _update_children(v['children']) h[p] = v return h def hashtree(path, ignore_empty_folder=False, ignore=None): """Given a path and options, compute the hash's upper tree. This is not for production use. It's merely a helper function used mainly in bin/swh-hashtree Args: - path: The path to hash - ignore_empty_folder: An option to ignore empty folder - ignore: An option to ignore patterns in directory names. Returns: The path's checksums respecting the options passed as parameters. """ if os.path.exists(path): if not os.path.isdir(path): raise ValueError('%s should be a directory!' % path) else: raise ValueError('%s should exist!' % path) if isinstance(path, str): path = path.encode('utf-8') if ignore: patterns = [] for exc in ignore: patterns.append(exc.encode('utf-8')) def dir_ok_fn_basic(dirpath, patterns=patterns): dname = os.path.basename(dirpath) for pattern_to_ignore in patterns: if pattern_to_ignore == dname: return False if (pattern_to_ignore + b'/') in dirpath: return False return True if ignore_empty_folder: def dir_ok_fn(dirpath, patterns=patterns): if not dir_ok_fn_basic(dirpath): return False return os.listdir(dirpath) != [] else: dir_ok_fn = dir_ok_fn_basic else: if ignore_empty_folder: def dir_ok_fn(dirpath): return os.listdir(dirpath) != [] else: dir_ok_fn = git.default_validation_dir objects = git.compute_hashes_from_directory( path, dir_ok_fn=dir_ok_fn) h = objects[path]['checksums'] return h def init_svn_repo_from_archive_dump(archive_path, root_temp_dir='/tmp'): """Given a path to an archive containing an svn dump. Initialize an svn repository with the content of said dump. Returns: A tuple: - temporary folder: containing the mounted repository - repo_path, path to the mounted repository inside the temporary folder Raises: ValueError in case of failure to run the command to uncompress and load the dump. """ project_name = os.path.basename(os.path.dirname(archive_path)) temp_dir = tempfile.mkdtemp(suffix='.swh.loader.svn', prefix='tmp.', dir=root_temp_dir) try: repo_path = os.path.join(temp_dir, project_name) # create the repository that will be loaded with the dump cmd = ['svnadmin', 'create', repo_path] r = call(cmd) if r != 0: raise ValueError( 'Failed to initialize empty svn repo for %s' % project_name) - with Popen(['pigz', '-dc', archive_path], stdout=PIPE) as dump: + with Popen(['gzip', '-dc', archive_path], stdout=PIPE) as dump: cmd = ['svnadmin', 'load', '-q', repo_path] r = call(cmd, stdin=dump.stdout) if r != 0: raise ValueError( 'Failed to mount the svn dump for project %s' % project_name) return temp_dir, repo_path except Exception as e: shutil.rmtree(temp_dir) raise e diff --git a/version.txt b/version.txt index 26249a9..5f0cdff 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.24-0-gffae957 \ No newline at end of file +v0.0.25-0-g6db09cb \ No newline at end of file