diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py index 4ccb1ce..14e5e4e 100644 --- a/swh/loader/svn/loader.py +++ b/swh/loader/svn/loader.py @@ -1,386 +1,388 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Loader in charge of injecting either new or existing svn mirrors to swh-storage. """ import datetime from swh.core import utils from swh.model import git, hashutil from swh.model.git import GitType from swh.loader.core.loader import SWHLoader from . import svn, converters class BaseSvnLoader(SWHLoader): """Base Svn loader to load one svn repository. There exists 2 different policies: - git-svn one (not for production): cf. GitSvnSvnLoader - SWH one: cf. SWHSvnLoader The main entry point of this is (no need to override it) - def load(self): Inherit this class and then override the following functions: - def build_swh_revision(self, rev, commit, dir_id, parents) This is in charge of converting an svn revision to a compliant swh revision - def process_repository(self) This is in charge of processing the actual svn repository and store the result to swh storage. """ CONFIG_BASE_FILENAME = 'loader/svn.ini' def __init__(self, svn_url, destination_path, origin, with_svn_update=True): super().__init__(origin['id'], logging_class='swh.loader.svn.SvnLoader') self.with_svn_update = with_svn_update # noqa self.origin = origin def build_swh_revision(self, rev, commit, dir_id, parents): """Convert an svn revision to an swh one according to the loader's policy (git-svn or swh). Args: rev: the svn revision number commit: dictionary with keys 'author_name', 'author_date', 'rev', 'message' dir_id: the hash tree computation parents: the revision's parents Returns: The swh revision """ raise NotImplementedError('This should be overriden by subclass') def process_repository(self): """The main idea of this function is to: - iterate over the svn commit logs - extract the svn commit log metadata - compute the hashes from the current directory down to the file - compute the equivalent swh revision - send all those objects for storage - create an swh occurrence pointing to the last swh revision seen - send that occurrence for storage in swh-storage. """ raise NotImplementedError('This should be implemented in subclass.') def process_svn_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process revisions from revision_start to revision_end and send to swh for storage. At each svn revision, checkout the repository, compute the tree hash and blobs and send for swh storage to store. Then computes and yields the swh revision. Yields: swh revision """ gen_revs = svnrepo.swh_hash_data_per_revision( revision_start, revision_end) for rev, nextrev, commit, objects_per_path in gen_revs: # compute the fs tree's checksums dir_id = objects_per_path[b'']['checksums']['sha1_git'] swh_revision = self.build_swh_revision( rev, commit, dir_id, revision_parents[rev]) swh_revision['id'] = git.compute_revision_sha1_git(swh_revision) self.log.debug('rev: %s, swhrev: %s, dir: %s' % ( rev, hashutil.hash_to_hex(swh_revision['id']), hashutil.hash_to_hex(dir_id))) if nextrev: revision_parents[nextrev] = [swh_revision['id']] self.maybe_load_contents( git.objects_per_type(GitType.BLOB, objects_per_path)) self.maybe_load_directories( git.objects_per_type(GitType.TREE, objects_per_path)) yield swh_revision def process_swh_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process and store revision to swh (sent by by blocks of 'revision_packet_size') Returns: The latest revision stored. """ for revisions in utils.grouper( self.process_svn_revisions(svnrepo, revision_start, revision_end, revision_parents), self.config['revision_packet_size']): revs = list(revisions) self.log.info('Processed %s revisions: [%s, ...]' % ( len(revs), hashutil.hash_to_hex(revs[0]['id']))) self.maybe_load_revisions(revs) return revs[-1] def process_swh_occurrence(self, revision, origin): """Process and load the occurrence pointing to the latest revision. """ occ = converters.build_swh_occurrence(revision['id'], origin['id'], datetime.datetime.utcnow()) self.log.debug('occ: %s' % occ) self.maybe_load_occurrences([occ]) def load(self): """Load a svn repository in swh. Checkout the svn repository locally in destination_path. Args: - svn_url: svn repository url to import - origin: Dictionary origin - id: origin's id - url: url origin we fetched - type: type of the origin Returns: Dictionary with the following keys: - status: mandatory, the status result as a boolean - stderr: optional when status is True, mandatory otherwise """ try: self.process_repository() finally: # flush eventual remaining data self.flush() self.svnrepo.clean_fs() return {'status': True} class GitSvnSvnLoader(BaseSvnLoader): """Git-svn like loader (compute hashes a-la git-svn) Notes: This implementation is: - NOT for production - NOT able to deal with update. Default policy: Its default policy is to enrich (or even alter) information at each svn revision. It will: - truncate the timestamp of the svn commit date - alter the user to be an email using the repository's uuid as mailserver (user -> user@) - fills in the gap for empty author with '(no author)' name - remove empty folder (thus not counting them during hash computation) The equivalent git command is: `git svn clone -q --no-metadata` """ def __init__(self, svn_url, destination_path, origin): super().__init__(svn_url, destination_path, origin, with_svn_update=False) # We don't want to persist result in git-svn policy self.config['send_contents'] = False self.config['send_directories'] = False self.config['send_revisions'] = False self.config['send_releases'] = False self.config['send_occurrences'] = False self.svnrepo = svn.GitSvnSvnRepo( svn_url, origin['id'], self.storage, destination_path=destination_path) def build_swh_revision(self, rev, commit, dir_id, parents): """Build the swh revision a-la git-svn. Args: rev: the svn revision commit: the commit metadata dir_id: the upper tree's hash identifier parents: the parents' identifiers Returns: The swh revision corresponding to the svn revision without any extra headers. """ return converters.build_gitsvn_swh_revision(rev, commit, dir_id, parents) def process_repository(self): """Load the repository's commits and send them for storage to swh. This does not deal with update. """ origin = self.origin svnrepo = self.svnrepo # default configuration revision_start = 1 revision_parents = { revision_start: [] } revision_end = svnrepo.head_revision() self.log.info('[revision_start-revision_end]: [%s-%s]' % ( revision_start, revision_end)) if revision_start == revision_end and revision_start is not 1: self.log.info('%s@%s already injected.' % ( svnrepo.remote_url, revision_end)) return {'status': True} self.log.info('Processing %s.' % svnrepo) # process and store revision to swh (sent by by blocks of # 'revision_packet_size') latest_rev = self.process_swh_revisions(svnrepo, revision_start, revision_end, revision_parents) self.process_swh_occurrence(latest_rev, origin) class SWHSvnLoader(BaseSvnLoader): """Swh svn loader is the main implementation destined for production. This implementation is able to deal with update on known svn repository. Default policy: It's to not add any information and be as close as possible from the svn data the server sent its way. The only thing that are added are the swh's revision 'extra_header' to be able to deal with update. """ - def __init__(self, svn_url, destination_path, origin): - super().__init__(svn_url, destination_path, origin) + def __init__(self, svn_url, destination_path, origin, + with_svn_update=True): + super().__init__(svn_url, destination_path, origin, with_svn_update) self.svnrepo = svn.SWHSvnRepo( svn_url, origin['id'], self.storage, destination_path=destination_path) def check_history_not_altered(self, svnrepo, revision_start, swh_rev): """Given a svn repository, check if the history was not tampered with. """ revision_id = swh_rev['id'] parents = swh_rev['parents'] hash_data_per_revs = svnrepo.swh_hash_data_at_revision(revision_start) rev = revision_start rev, _, commit, objects_per_path = list(hash_data_per_revs)[0] dir_id = objects_per_path[b'']['checksums']['sha1_git'] swh_revision = self.build_swh_revision(rev, commit, dir_id, parents) swh_revision_id = git.compute_revision_sha1_git(swh_revision) return swh_revision_id == revision_id def build_swh_revision(self, rev, commit, dir_id, parents): """Build the swh revision dictionary. This adds: - the 'synthetic' flag to true - the 'extra_headers' containing the repository's uuid and the svn revision number. Args: rev: the svn revision commit: the commit metadata dir_id: the upper tree's hash identifier parents: the parents' identifiers Returns: The swh revision corresponding to the svn revision. """ return converters.build_swh_revision(rev, commit, self.svnrepo.uuid, dir_id, parents) def process_repository(self): svnrepo = self.svnrepo origin = self.origin # default configuration revision_start = 1 revision_parents = { revision_start: [] } # Deal with update - swh_rev = svnrepo.swh_previous_revision() - - if swh_rev: # Yes, we do. Try and update it. - extra_headers = dict(swh_rev['metadata']['extra_headers']) - revision_start = int(extra_headers['svn_revision']) - revision_parents = { - revision_start: swh_rev['parents'] - } - - self.log.debug('svn co %s@%s' % (svnrepo.remote_url, - revision_start)) - - if swh_rev and not self.check_history_not_altered( - svnrepo, - revision_start, - swh_rev): - msg = 'History of svn %s@%s history modified. Skipping...' % ( # noqa - svnrepo.remote_url, revision_start) - self.log.warn(msg) - return {'status': False, 'stderr': msg} + if self.with_svn_update: + swh_rev = svnrepo.swh_previous_revision() + + if swh_rev: # Yes, we do. Try and update it. + extra_headers = dict(swh_rev['metadata']['extra_headers']) + revision_start = int(extra_headers['svn_revision']) + revision_parents = { + revision_start: swh_rev['parents'] + } + + self.log.debug('svn co %s@%s' % (svnrepo.remote_url, + revision_start)) + + if swh_rev and not self.check_history_not_altered( + svnrepo, + revision_start, + swh_rev): + msg = 'History of svn %s@%s history modified. Skipping...' % ( # noqa + svnrepo.remote_url, revision_start) + self.log.warn(msg) + return {'status': False, 'stderr': msg} revision_end = svnrepo.head_revision() self.log.info('[revision_start-revision_end]: [%s-%s]' % ( revision_start, revision_end)) if revision_start == revision_end and revision_start is not 1: self.log.info('%s@%s already injected.' % ( svnrepo.remote_url, revision_end)) return {'status': True} self.log.info('Processing %s.' % svnrepo) # process and store revision to swh (sent by by blocks of # 'revision_packet_size') latest_rev = self.process_swh_revisions(svnrepo, revision_start, revision_end, revision_parents) self.process_swh_occurrence(latest_rev, origin) diff --git a/swh/loader/svn/tests/test_converters.org b/swh/loader/svn/tests/test_converters.org new file mode 100644 index 0000000..1b79756 --- /dev/null +++ b/swh/loader/svn/tests/test_converters.org @@ -0,0 +1,112 @@ +#+title: Prepare test_converters.py it tests +#+author: ardumont + +* Requisite: + +#+BEGIN_SRC sh +sudo apt install subversion git-svn +#+END_SRC + +* Create mirror repository + +Then: +#+BEGIN_SRC sh +./init-svn-repository.sh /home/storage/svn/repos/pkg-gourmet svn://svn.debian.org/svn/pkg-gourmet/ +#+END_SRC +Note: +Saved as ../../../../bin/init-svn-repository.sh + +And now we have a mirror svn repository at file:///home/storage/svn/repos/pkg-gourmet + +* git-svn policy + +`git svn clone` the repository and parse the git log entries for the needed data. + +#+BEGIN_SRC sh +git svn clone file:///home/storage/svn/repos/pkg-gourmet -q --no-metadata +cd pkg-gourmet +# commit +git log --format=raw --reverse | grep '^commit ' | awk '{print $2}' +# tree +git log --format=raw --reverse | grep '^tree ' | awk '{print $2}' +#+END_SRC + +Those are the data to check when done parsing the repository: + +|------------------------------------------+------------------------------------------| +| revision | tree | +|------------------------------------------+------------------------------------------| +| 22c0fa5195a53f2e733ec75a9b6e9d1624a8b771 | 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | +| 17a631d474f49bbebfdf3d885dcde470d7faafd7 | 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | +| c8a9172b2a615d461154f61158180de53edc6070 | 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | +| 7c8f83394b6e8966eb46f0d3416c717612198a4b | 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | +| 852547b3b2bb76c8582cee963e8aa180d552a15c | ab047e38d1532f61ff5c3621202afc3e763e9945 | +| bad4a83737f337d47e0ba681478214b07a707218 | 9bcfc25001b71c333b4b5a89224217de81c56e2e | +|------------------------------------------+------------------------------------------| + +* swh policy + +For this one this was more tedious. +#+BEGIN_SRC sh +$ svn export --ignore-keywords file:///home/storage/svn/repos/pkg-gourmet@1 +#+END_SRC +The export does not expand the keywords and does not include the .svn folder. + +Then: +#+BEGIN_SRC sh +$ cd pkg-gourmet +$ swh-hashtree --path . +669a71cce6c424a81ba42b7dc5d560d32252f0ca +#+END_SRC + +Note: ../../../../bin/hashtree + +Then for the next revision: +#+BEGIN_SRC sh +cd .. ; rm -rf pkg-gourmet; svn export --ignore-keywords file:///home/storage/svn/repos/pkg-gourmet@2 +A pkg-gourmet +A pkg-gourmet/gourmet +A pkg-gourmet/gourmet/trunk +Exported revision 2. +$ cd pkg-gourmet && swh-hashtree --path . +008ac97a1118560797c50e3392fa1443acdaa349 +#+END_SRC +etc... + +|------------------------------------------+------------------------------------------| +| revision | tree | +|------------------------------------------+------------------------------------------| +| 0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71 | 669a71cce6c424a81ba42b7dc5d560d32252f0ca | +| 95edacc8848369d6fb1608e887d6d2474fd5224f | 008ac97a1118560797c50e3392fa1443acdaa349 | +| fef26ea45a520071711ba2b9d16a2985ee837021 | 3780effbe846a26751a95a8c95c511fb72be15b4 | +| 3f51abf3b3d466571be0855dfa67e094f9ceff1b | ffcca9b09c5827a6b8137322d4339c8055c3ee1e | +| a3a577948fdbda9d1061913b77a1588695eadb41 | 7dc52cc04c3b8bd7c085900d60c159f7b846f866 | +| 4876cb10aec6f708f7466dddf547567b65f6c39c | 0deab3023ac59398ae467fc4bff5583008af1ee2 | +|------------------------------------------+------------------------------------------| + +For the revision, cheating a little. +That is adapting swh.model.identifiers.revision_identifiers to print the commit's manifest: + +#+BEGIN_SRC sh +b'tree 669a71cce6c424a81ba42b7dc5d560d32252f0ca\nauthor seanius 1138341038.645397 +0000\ncommitter seanius 1138341038.645397 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 1\n\nmaking dir structure...' +[2016-06-23 12:35:39,291: DEBUG/Worker-1] rev: 1, swhrev: 0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71, dir: 669a71cce6c424a81ba42b7dc5d560d32252f0ca +b'tree 008ac97a1118560797c50e3392fa1443acdaa349\nparent 0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71\nauthor seanius 1138341044.821526 +0000\ncommitter seanius 1138341044.821526 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 2\n\nmaking dir structure...' +[2016-06-23 12:35:39,302: DEBUG/Worker-1] rev: 2, swhrev: 95edacc8848369d6fb1608e887d6d2474fd5224f, dir: 008ac97a1118560797c50e3392fa1443acdaa349 +b'tree 3780effbe846a26751a95a8c95c511fb72be15b4\nparent 95edacc8848369d6fb1608e887d6d2474fd5224f\nauthor seanius 1138341057.282488 +0000\ncommitter seanius 1138341057.282488 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 3\n\nmaking dir structure...' +[2016-06-23 12:35:39,313: DEBUG/Worker-1] rev: 3, swhrev: fef26ea45a520071711ba2b9d16a2985ee837021, dir: 3780effbe846a26751a95a8c95c511fb72be15b4 +b'tree ffcca9b09c5827a6b8137322d4339c8055c3ee1e\nparent fef26ea45a520071711ba2b9d16a2985ee837021\nauthor seanius 1138341064.191867 +0000\ncommitter seanius 1138341064.191867 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 4\n\nmaking dir structure...' +[2016-06-23 12:35:39,322: DEBUG/Worker-1] rev: 4, swhrev: 3f51abf3b3d466571be0855dfa67e094f9ceff1b, dir: ffcca9b09c5827a6b8137322d4339c8055c3ee1e +b'tree 7dc52cc04c3b8bd7c085900d60c159f7b846f866\nparent 3f51abf3b3d466571be0855dfa67e094f9ceff1b\nauthor seanius 1138342632.066765 +0000\ncommitter seanius 1138342632.066765 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 5\n\ninitial import' +[2016-06-23 12:35:39,339: DEBUG/Worker-1] rev: 5, swhrev: a3a577948fdbda9d1061913b77a1588695eadb41, dir: 7dc52cc04c3b8bd7c085900d60c159f7b846f866 +b'tree 0deab3023ac59398ae467fc4bff5583008af1ee2\nparent a3a577948fdbda9d1061913b77a1588695eadb41\nauthor seanius 1138343905.448277 +0000\ncommitter seanius 1138343905.448277 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 6\n\nfix breakage in rules' +[2016-06-23 12:35:39,348: DEBUG/Worker-1] rev: 6, swhrev: 4876cb10aec6f708f7466dddf547567b65f6c39c, dir: 0deab3023ac59398ae467fc4bff5583008af1ee2 +[2016-06-23 12:35:39,355: INFO/Worker-1] Processed 6 revisions: [0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71, ...] +#+END_SRC + +Then checking the manifest's hash is ok: +#+BEGIN_SRC sh +$ echo -en 'tree 669a71cce6c424a81ba42b7dc5d560d32252f0ca\nauthor seanius 1138341038.645397 +0000\ncommitter seanius 1138341038.645397 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 1\n\nmaking dir structure...' | git hash-object -t commit --stdin +0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71 +#+END_SRC + +And all is ok. diff --git a/swh/loader/svn/tests/test_loader.py b/swh/loader/svn/tests/test_loader.py new file mode 100644 index 0000000..feefc7c --- /dev/null +++ b/swh/loader/svn/tests/test_loader.py @@ -0,0 +1,197 @@ +# Copyright (C) 2016 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import os +import shutil +import subprocess +import tempfile +import unittest + +from nose.tools import istest + +from swh.core import hashutil +from swh.loader.svn.loader import GitSvnSvnLoader, SWHSvnLoader + + +# Define loaders with no storage +# They'll just accumulate the data in place +# Only for testing purposes. + +class GitSvnLoaderNoStorage(GitSvnSvnLoader): + """A GitSvnLoader with no persistence. + + """ + def __init__(self, svn_url, destination_path, origin): + super().__init__(svn_url, destination_path, origin) + self.all_contents = [] + self.all_directories = [] + self.all_revisions = [] + self.all_releases = [] + self.all_occurrences = [] + + def maybe_load_contents(self, all_contents): + self.all_contents.extend(all_contents) + + def maybe_load_directories(self, all_directories): + self.all_directories.extend(all_directories) + + def maybe_load_revisions(self, all_revisions): + self.all_revisions.extend(all_revisions) + + def maybe_load_releases(self, releases): + raise ValueError('If called, the test must break.') + + def maybe_load_occurrences(self, all_occurrences): + self.all_occurrences.extend(all_occurrences) + + +class SWHSvnLoaderNoStorage(SWHSvnLoader): + """An SWHSVNLoader with no persistence. + + """ + def __init__(self, svn_url, destination_path, origin): + super().__init__(svn_url, destination_path, origin, + with_svn_update=False) + # We don't want to persist any result in this test context + self.config['send_contents'] = False + self.config['send_directories'] = False + self.config['send_revisions'] = False + self.config['send_releases'] = False + self.config['send_occurrences'] = False + self.all_contents = [] + self.all_directories = [] + self.all_revisions = [] + self.all_releases = [] + self.all_occurrences = [] + + def maybe_load_contents(self, all_contents): + self.all_contents.extend(all_contents) + + def maybe_load_directories(self, all_directories): + self.all_directories.extend(all_directories) + + def maybe_load_revisions(self, all_revisions): + self.all_revisions.extend(all_revisions) + + def maybe_load_releases(self, releases): + raise ValueError('If called, the test must break.') + + def maybe_load_occurrences(self, all_occurrences): + self.all_occurrences.extend(all_occurrences) + + +class BaseTestLoader(unittest.TestCase): + def setUp(self): + super().setUpClass() + + self.tmp_root_path = tempfile.mkdtemp() + + start_path = os.path.dirname(__file__) + filename = 'pkg-gourmet' + svn_mirror_repo = os.path.join(start_path, + '../../../../..', + 'swh-storage-testdata', + 'svn-folders', + filename + '.tgz') + + # uncompress the sample folder + subprocess.check_output( + ['tar', 'xvf', svn_mirror_repo, '-C', self.tmp_root_path], + ) + + self.svn_mirror_url = 'file://' + self.tmp_root_path + '/' + filename + self.destination_path = os.path.join( + self.tmp_root_path, 'working-copy') + + def tearDown(self): + super().tearDownClass() + + shutil.rmtree(self.tmp_root_path) + + +class GitSvnLoaderITTest(BaseTestLoader): + def setUp(self): + super().setUp() + + self.loader = GitSvnLoaderNoStorage( + svn_url=self.svn_mirror_url, + destination_path=self.destination_path, + origin={'id': 1, 'type': 'svn', 'url': 'file:///dev/null'}) + + @istest + def process_repository(self): + """Process repository the git-svn way should be ok.""" + # when + self.loader.process_repository() + + # then + self.assertEquals(len(self.loader.all_revisions), 6) + self.assertEquals(len(self.loader.all_releases), 0) + self.assertEquals(len(self.loader.all_occurrences), 1) + + # cf. test_converters.org for explaining from where those hash + # come from + expected_revisions = { + # revision hash | directory hash # noqa + '22c0fa5195a53f2e733ec75a9b6e9d1624a8b771': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', # noqa + '17a631d474f49bbebfdf3d885dcde470d7faafd7': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', # noqa + 'c8a9172b2a615d461154f61158180de53edc6070': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', # noqa + '7c8f83394b6e8966eb46f0d3416c717612198a4b': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', # noqa + '852547b3b2bb76c8582cee963e8aa180d552a15c': 'ab047e38d1532f61ff5c3621202afc3e763e9945', # noqa + 'bad4a83737f337d47e0ba681478214b07a707218': '9bcfc25001b71c333b4b5a89224217de81c56e2e', # noqa + } + + for rev in self.loader.all_revisions: + rev_id = hashutil.hash_to_hex(rev['id']) + directory_id = hashutil.hash_to_hex(rev['directory']) + + self.assertEquals(expected_revisions[rev_id], directory_id) + + occ = self.loader.all_occurrences[0] + self.assertEquals(hashutil.hash_to_hex(occ['target']), + 'bad4a83737f337d47e0ba681478214b07a707218') + + +class SWHSvnLoaderITTest(BaseTestLoader): + def setUp(self): + super().setUp() + + self.loader = SWHSvnLoaderNoStorage( + svn_url=self.svn_mirror_url, + destination_path=self.destination_path, + origin={'id': 2, 'type': 'svn', 'url': 'file:///dev/null'}) + + @istest + def process_repository(self): + """Process repository the swh way should be ok.""" + # when + self.loader.process_repository() + + # then + self.assertEquals(len(self.loader.all_revisions), 6) + self.assertEquals(len(self.loader.all_releases), 0) + self.assertEquals(len(self.loader.all_occurrences), 1) + + # cf. test_converters.org for explaining from where those hash + # come from + expected_revisions = { + # revision hash | directory hash + '0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71': '669a71cce6c424a81ba42b7dc5d560d32252f0ca', # noqa + '95edacc8848369d6fb1608e887d6d2474fd5224f': '008ac97a1118560797c50e3392fa1443acdaa349', # noqa + 'fef26ea45a520071711ba2b9d16a2985ee837021': '3780effbe846a26751a95a8c95c511fb72be15b4', # noqa + '3f51abf3b3d466571be0855dfa67e094f9ceff1b': 'ffcca9b09c5827a6b8137322d4339c8055c3ee1e', # noqa + 'a3a577948fdbda9d1061913b77a1588695eadb41': '7dc52cc04c3b8bd7c085900d60c159f7b846f866', # noqa + '4876cb10aec6f708f7466dddf547567b65f6c39c': '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa + } + + for rev in self.loader.all_revisions: + rev_id = hashutil.hash_to_hex(rev['id']) + directory_id = hashutil.hash_to_hex(rev['directory']) + + self.assertEquals(expected_revisions[rev_id], directory_id) + + occ = self.loader.all_occurrences[0] + self.assertEquals(hashutil.hash_to_hex(occ['target']), + '4876cb10aec6f708f7466dddf547567b65f6c39c')