diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py index d7e57a1..b219bea 100644 --- a/swh/loader/svn/loader.py +++ b/swh/loader/svn/loader.py @@ -1,392 +1,399 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Loader in charge of injecting either new or existing svn mirrors to swh-storage. """ import datetime from swh.core import utils from swh.model import git, hashutil from swh.model.git import GitType from swh.loader.core.loader import SWHLoader from . import svn, converters class BaseSvnLoader(SWHLoader): """Base Svn loader to load one svn repository. There exists 2 different policies: - git-svn one (not for production): cf. GitSvnSvnLoader - SWH one: cf. SWHSvnLoader The main entry point of this is (no need to override it) - def load(self): Inherit this class and then override the following functions: - def build_swh_revision(self, rev, commit, dir_id, parents) This is in charge of converting an svn revision to a compliant swh revision - def process_repository(self) This is in charge of processing the actual svn repository and store the result to swh storage. """ CONFIG_BASE_FILENAME = 'loader/svn.ini' def __init__(self, svn_url, destination_path, origin, with_svn_update=True): super().__init__(origin['id'], logging_class='swh.loader.svn.SvnLoader') self.with_svn_update = with_svn_update # noqa self.origin = origin def build_swh_revision(self, rev, commit, dir_id, parents): """Convert an svn revision to an swh one according to the loader's policy (git-svn or swh). Args: rev: the svn revision number commit: dictionary with keys 'author_name', 'author_date', 'rev', 'message' dir_id: the hash tree computation parents: the revision's parents Returns: The swh revision """ raise NotImplementedError('This should be overriden by subclass') def process_repository(self): """The main idea of this function is to: - iterate over the svn commit logs - extract the svn commit log metadata - compute the hashes from the current directory down to the file - compute the equivalent swh revision - send all those objects for storage - create an swh occurrence pointing to the last swh revision seen - send that occurrence for storage in swh-storage. """ raise NotImplementedError('This should be implemented in subclass.') def process_svn_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process revisions from revision_start to revision_end and send to swh for storage. At each svn revision, checkout the repository, compute the tree hash and blobs and send for swh storage to store. Then computes and yields the swh revision. Yields: swh revision """ gen_revs = svnrepo.swh_hash_data_per_revision( revision_start, revision_end) for rev, nextrev, commit, objects_per_path in gen_revs: # compute the fs tree's checksums dir_id = objects_per_path[b'']['checksums']['sha1_git'] swh_revision = self.build_swh_revision( rev, commit, dir_id, revision_parents[rev]) swh_revision['id'] = git.compute_revision_sha1_git(swh_revision) self.log.debug('rev: %s, swhrev: %s, dir: %s' % ( rev, hashutil.hash_to_hex(swh_revision['id']), hashutil.hash_to_hex(dir_id))) if nextrev: revision_parents[nextrev] = [swh_revision['id']] self.maybe_load_contents( git.objects_per_type(GitType.BLOB, objects_per_path)) self.maybe_load_directories( git.objects_per_type(GitType.TREE, objects_per_path)) yield swh_revision def process_swh_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process and store revision to swh (sent by by blocks of 'revision_packet_size') Returns: The latest revision stored. """ for revisions in utils.grouper( self.process_svn_revisions(svnrepo, revision_start, revision_end, revision_parents), self.config['revision_packet_size']): revs = list(revisions) self.log.info('Processed %s revisions: [%s, ...]' % ( len(revs), hashutil.hash_to_hex(revs[0]['id']))) self.maybe_load_revisions(revs) return revs[-1] def process_swh_occurrence(self, revision, origin): """Process and load the occurrence pointing to the latest revision. """ occ = converters.build_swh_occurrence(revision['id'], origin['id'], datetime.datetime.utcnow()) self.log.debug('occ: %s' % occ) self.maybe_load_occurrences([occ]) def load(self): """Load a svn repository in swh. Checkout the svn repository locally in destination_path. Args: - svn_url: svn repository url to import - origin: Dictionary origin - id: origin's id - url: url origin we fetched - type: type of the origin Returns: Dictionary with the following keys: - status: mandatory, the status result as a boolean - stderr: optional when status is True, mandatory otherwise """ try: self.process_repository() finally: # flush eventual remaining data self.flush() self.svnrepo.clean_fs() return {'status': True} class GitSvnSvnLoader(BaseSvnLoader): """Git-svn like loader (compute hashes a-la git-svn) Notes: This implementation is: - NOT for production - NOT able to deal with update. Default policy: Its default policy is to enrich (or even alter) information at each svn revision. It will: - truncate the timestamp of the svn commit date - alter the user to be an email using the repository's uuid as mailserver (user -> user@) - fills in the gap for empty author with '(no author)' name - remove empty folder (thus not counting them during hash computation) The equivalent git command is: `git svn clone -q --no-metadata` """ def __init__(self, svn_url, destination_path, origin): super().__init__(svn_url, destination_path, origin) # We don't want to persist result in git-svn policy self.config['send_contents'] = False self.config['send_directories'] = False self.config['send_revisions'] = False self.config['send_releases'] = False self.config['send_occurrences'] = False self.svnrepo = svn.GitSvnSvnRepo( svn_url, origin['id'], self.storage, destination_path=destination_path) def build_swh_revision(self, rev, commit, dir_id, parents): """Build the swh revision a-la git-svn. Args: rev: the svn revision commit: the commit metadata dir_id: the upper tree's hash identifier parents: the parents' identifiers Returns: The swh revision corresponding to the svn revision without any extra headers. """ return converters.build_gitsvn_swh_revision(rev, commit, dir_id, parents) def process_repository(self): """Load the repository's commits and send them for storage to swh. This does not deal with update. """ origin = self.origin svnrepo = self.svnrepo # default configuration revision_start = 1 revision_parents = { revision_start: [] } revision_end = svnrepo.head_revision() self.log.info('[revision_start-revision_end]: [%s-%s]' % ( revision_start, revision_end)) if revision_start == revision_end and revision_start is not 1: self.log.info('%s@%s already injected.' % ( svnrepo.remote_url, revision_end)) return {'status': True} self.log.info('Processing %s.' % svnrepo) # process and store revision to swh (sent by by blocks of # 'revision_packet_size') latest_rev = self.process_swh_revisions(svnrepo, revision_start, revision_end, revision_parents) self.process_swh_occurrence(latest_rev, origin) class SWHSvnLoader(BaseSvnLoader): """Swh svn loader is the main implementation destined for production. This implementation is able to deal with update on known svn repository. Default policy: It's to not add any information and be as close as possible from the svn data the server sent its way. The only thing that are added are the swh's revision 'extra_header' to be able to deal with update. """ def __init__(self, svn_url, destination_path, origin, with_svn_update=True): super().__init__(svn_url, destination_path, origin, with_svn_update) self.svnrepo = svn.SWHSvnRepo( svn_url, origin['id'], self.storage, destination_path=destination_path) def swh_previous_revision(self): """Retrieve swh's previous revision if any. """ return self.svnrepo.swh_previous_revision() def check_history_not_altered(self, svnrepo, revision_start, swh_rev): """Given a svn repository, check if the history was not tampered with. """ revision_id = swh_rev['id'] parents = swh_rev['parents'] hash_data_per_revs = svnrepo.swh_hash_data_at_revision(revision_start) rev = revision_start rev, _, commit, objects_per_path = list(hash_data_per_revs)[0] dir_id = objects_per_path[b'']['checksums']['sha1_git'] swh_revision = self.build_swh_revision(rev, commit, dir_id, parents) swh_revision_id = git.compute_revision_sha1_git(swh_revision) return swh_revision_id == revision_id def build_swh_revision(self, rev, commit, dir_id, parents): """Build the swh revision dictionary. This adds: - the 'synthetic' flag to true - the 'extra_headers' containing the repository's uuid and the svn revision number. Args: rev: the svn revision commit: the commit metadata dir_id: the upper tree's hash identifier parents: the parents' identifiers Returns: The swh revision corresponding to the svn revision. """ return converters.build_swh_revision(rev, commit, self.svnrepo.uuid, dir_id, parents) def process_repository(self): svnrepo = self.svnrepo origin = self.origin # default configuration revision_start = 1 revision_parents = { revision_start: [] } # Deal with update swh_rev = self.swh_previous_revision() if swh_rev: # Yes, we do. Try and update it. extra_headers = dict(swh_rev['metadata']['extra_headers']) revision_start = int(extra_headers['svn_revision']) revision_parents = { revision_start: swh_rev['parents'] } - self.log.debug('svn co %s@%s' % (svnrepo.remote_url, - revision_start)) + self.log.debug('svn export --ignore-keywords %s@%s' % ( + svnrepo.remote_url, + revision_start)) if swh_rev and not self.check_history_not_altered( svnrepo, revision_start, swh_rev): msg = 'History of svn %s@%s history modified. Skipping...' % ( # noqa svnrepo.remote_url, revision_start) self.log.warn(msg) return {'status': False, 'stderr': msg} + else: + # now we know history is ok, we start at next revision + revision_start = revision_start + 1 + # and the parent become the latest know revision for + # that repository + revision_parents[revision_start] = [swh_rev['id']] revision_end = svnrepo.head_revision() self.log.info('[revision_start-revision_end]: [%s-%s]' % ( revision_start, revision_end)) - if revision_start == revision_end and revision_start is not 1: + if revision_start >= revision_end and revision_start is not 1: self.log.info('%s@%s already injected.' % ( svnrepo.remote_url, revision_end)) return {'status': True} self.log.info('Processing %s.' % svnrepo) # process and store revision to swh (sent by by blocks of # 'revision_packet_size') latest_rev = self.process_swh_revisions(svnrepo, revision_start, revision_end, revision_parents) self.process_swh_occurrence(latest_rev, origin) diff --git a/swh/loader/svn/tests/test_loader.org b/swh/loader/svn/tests/test_loader.org index 1b79756..5537487 100644 --- a/swh/loader/svn/tests/test_loader.org +++ b/swh/loader/svn/tests/test_loader.org @@ -1,112 +1,284 @@ #+title: Prepare test_converters.py it tests #+author: ardumont * Requisite: #+BEGIN_SRC sh sudo apt install subversion git-svn #+END_SRC * Create mirror repository Then: #+BEGIN_SRC sh ./init-svn-repository.sh /home/storage/svn/repos/pkg-gourmet svn://svn.debian.org/svn/pkg-gourmet/ #+END_SRC Note: Saved as ../../../../bin/init-svn-repository.sh And now we have a mirror svn repository at file:///home/storage/svn/repos/pkg-gourmet * git-svn policy `git svn clone` the repository and parse the git log entries for the needed data. #+BEGIN_SRC sh git svn clone file:///home/storage/svn/repos/pkg-gourmet -q --no-metadata cd pkg-gourmet # commit git log --format=raw --reverse | grep '^commit ' | awk '{print $2}' # tree git log --format=raw --reverse | grep '^tree ' | awk '{print $2}' #+END_SRC Those are the data to check when done parsing the repository: |------------------------------------------+------------------------------------------| | revision | tree | |------------------------------------------+------------------------------------------| | 22c0fa5195a53f2e733ec75a9b6e9d1624a8b771 | 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | | 17a631d474f49bbebfdf3d885dcde470d7faafd7 | 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | | c8a9172b2a615d461154f61158180de53edc6070 | 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | | 7c8f83394b6e8966eb46f0d3416c717612198a4b | 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | | 852547b3b2bb76c8582cee963e8aa180d552a15c | ab047e38d1532f61ff5c3621202afc3e763e9945 | | bad4a83737f337d47e0ba681478214b07a707218 | 9bcfc25001b71c333b4b5a89224217de81c56e2e | |------------------------------------------+------------------------------------------| * swh policy +** New repository + For this one this was more tedious. #+BEGIN_SRC sh $ svn export --ignore-keywords file:///home/storage/svn/repos/pkg-gourmet@1 #+END_SRC The export does not expand the keywords and does not include the .svn folder. Then: #+BEGIN_SRC sh $ cd pkg-gourmet $ swh-hashtree --path . 669a71cce6c424a81ba42b7dc5d560d32252f0ca #+END_SRC Note: ../../../../bin/hashtree Then for the next revision: #+BEGIN_SRC sh cd .. ; rm -rf pkg-gourmet; svn export --ignore-keywords file:///home/storage/svn/repos/pkg-gourmet@2 A pkg-gourmet A pkg-gourmet/gourmet A pkg-gourmet/gourmet/trunk Exported revision 2. $ cd pkg-gourmet && swh-hashtree --path . 008ac97a1118560797c50e3392fa1443acdaa349 #+END_SRC etc... -|------------------------------------------+------------------------------------------| -| revision | tree | -|------------------------------------------+------------------------------------------| -| 0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71 | 669a71cce6c424a81ba42b7dc5d560d32252f0ca | -| 95edacc8848369d6fb1608e887d6d2474fd5224f | 008ac97a1118560797c50e3392fa1443acdaa349 | -| fef26ea45a520071711ba2b9d16a2985ee837021 | 3780effbe846a26751a95a8c95c511fb72be15b4 | -| 3f51abf3b3d466571be0855dfa67e094f9ceff1b | ffcca9b09c5827a6b8137322d4339c8055c3ee1e | -| a3a577948fdbda9d1061913b77a1588695eadb41 | 7dc52cc04c3b8bd7c085900d60c159f7b846f866 | -| 4876cb10aec6f708f7466dddf547567b65f6c39c | 0deab3023ac59398ae467fc4bff5583008af1ee2 | -|------------------------------------------+------------------------------------------| +|--------------+------------------------------------------+------------------------------------------| +| svn revision | swh revision | tree | +|--------------+------------------------------------------+------------------------------------------| +| 1 | 0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71 | 669a71cce6c424a81ba42b7dc5d560d32252f0ca | +| 2 | 95edacc8848369d6fb1608e887d6d2474fd5224f | 008ac97a1118560797c50e3392fa1443acdaa349 | +| 3 | fef26ea45a520071711ba2b9d16a2985ee837021 | 3780effbe846a26751a95a8c95c511fb72be15b4 | +| 4 | 3f51abf3b3d466571be0855dfa67e094f9ceff1b | ffcca9b09c5827a6b8137322d4339c8055c3ee1e | +| 5 | a3a577948fdbda9d1061913b77a1588695eadb41 | 7dc52cc04c3b8bd7c085900d60c159f7b846f866 | +| 6 | 4876cb10aec6f708f7466dddf547567b65f6c39c | 0deab3023ac59398ae467fc4bff5583008af1ee2 | +|--------------+------------------------------------------+------------------------------------------| For the revision, cheating a little. That is adapting swh.model.identifiers.revision_identifiers to print the commit's manifest: #+BEGIN_SRC sh b'tree 669a71cce6c424a81ba42b7dc5d560d32252f0ca\nauthor seanius 1138341038.645397 +0000\ncommitter seanius 1138341038.645397 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 1\n\nmaking dir structure...' [2016-06-23 12:35:39,291: DEBUG/Worker-1] rev: 1, swhrev: 0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71, dir: 669a71cce6c424a81ba42b7dc5d560d32252f0ca b'tree 008ac97a1118560797c50e3392fa1443acdaa349\nparent 0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71\nauthor seanius 1138341044.821526 +0000\ncommitter seanius 1138341044.821526 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 2\n\nmaking dir structure...' [2016-06-23 12:35:39,302: DEBUG/Worker-1] rev: 2, swhrev: 95edacc8848369d6fb1608e887d6d2474fd5224f, dir: 008ac97a1118560797c50e3392fa1443acdaa349 b'tree 3780effbe846a26751a95a8c95c511fb72be15b4\nparent 95edacc8848369d6fb1608e887d6d2474fd5224f\nauthor seanius 1138341057.282488 +0000\ncommitter seanius 1138341057.282488 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 3\n\nmaking dir structure...' [2016-06-23 12:35:39,313: DEBUG/Worker-1] rev: 3, swhrev: fef26ea45a520071711ba2b9d16a2985ee837021, dir: 3780effbe846a26751a95a8c95c511fb72be15b4 b'tree ffcca9b09c5827a6b8137322d4339c8055c3ee1e\nparent fef26ea45a520071711ba2b9d16a2985ee837021\nauthor seanius 1138341064.191867 +0000\ncommitter seanius 1138341064.191867 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 4\n\nmaking dir structure...' [2016-06-23 12:35:39,322: DEBUG/Worker-1] rev: 4, swhrev: 3f51abf3b3d466571be0855dfa67e094f9ceff1b, dir: ffcca9b09c5827a6b8137322d4339c8055c3ee1e b'tree 7dc52cc04c3b8bd7c085900d60c159f7b846f866\nparent 3f51abf3b3d466571be0855dfa67e094f9ceff1b\nauthor seanius 1138342632.066765 +0000\ncommitter seanius 1138342632.066765 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 5\n\ninitial import' [2016-06-23 12:35:39,339: DEBUG/Worker-1] rev: 5, swhrev: a3a577948fdbda9d1061913b77a1588695eadb41, dir: 7dc52cc04c3b8bd7c085900d60c159f7b846f866 b'tree 0deab3023ac59398ae467fc4bff5583008af1ee2\nparent a3a577948fdbda9d1061913b77a1588695eadb41\nauthor seanius 1138343905.448277 +0000\ncommitter seanius 1138343905.448277 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 6\n\nfix breakage in rules' [2016-06-23 12:35:39,348: DEBUG/Worker-1] rev: 6, swhrev: 4876cb10aec6f708f7466dddf547567b65f6c39c, dir: 0deab3023ac59398ae467fc4bff5583008af1ee2 [2016-06-23 12:35:39,355: INFO/Worker-1] Processed 6 revisions: [0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71, ...] #+END_SRC Then checking the manifest's hash is ok: #+BEGIN_SRC sh $ echo -en 'tree 669a71cce6c424a81ba42b7dc5d560d32252f0ca\nauthor seanius 1138341038.645397 +0000\ncommitter seanius 1138341038.645397 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 1\n\nmaking dir structure...' | git hash-object -t commit --stdin 0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71 #+END_SRC And all is ok. + +** Update existing repository + +Checkout on disk the repository and do some modifications on it: +#+BEGIN_SRC sh +$ svn co file:///home/storage/svn/repos/pkg-gourmet/ +A pkg-gourmet/gourmet +A pkg-gourmet/gourmet/branches +A pkg-gourmet/gourmet/tags +A pkg-gourmet/gourmet/trunk +A pkg-gourmet/gourmet/trunk/debian +A pkg-gourmet/gourmet/trunk/debian/patches +A pkg-gourmet/gourmet/trunk/debian/patches/00list +A pkg-gourmet/gourmet/trunk/debian/patches/01_printer_warning.dpatch +A pkg-gourmet/gourmet/trunk/debian/README.Maintainer +A pkg-gourmet/gourmet/trunk/debian/TODO +A pkg-gourmet/gourmet/trunk/debian/changelog +A pkg-gourmet/gourmet/trunk/debian/compat +A pkg-gourmet/gourmet/trunk/debian/control +A pkg-gourmet/gourmet/trunk/debian/copyright +A pkg-gourmet/gourmet/trunk/debian/dirs +A pkg-gourmet/gourmet/trunk/debian/docs +A pkg-gourmet/gourmet/trunk/debian/gourmet.1 +A pkg-gourmet/gourmet/trunk/debian/menu +A pkg-gourmet/gourmet/trunk/debian/postinst +A pkg-gourmet/gourmet/trunk/debian/postrm +A pkg-gourmet/gourmet/trunk/debian/prerm +A pkg-gourmet/gourmet/trunk/debian/recbox.xpm +A pkg-gourmet/gourmet/trunk/debian/rules +A pkg-gourmet/gourmet/trunk/debian/source.lintian-overrides +Checked out revision 6. +$ cd pkg-gourmet +$ mkdir foo/bar/ -p +$ em foo/bar/new-file +% svn add foo +A foo +A foo/bar +A foo/bar/README +$ svn commit -m 'Add a new README' +Adding foo +Adding foo/bar +Adding foo/bar/README +Transmitting file data .done +Committing transaction... +Committed revision 7. +$ ln -s foo/bar/README README +$ svn add README +A README +$ svn commit -m 'Add link to README' +Adding README +Transmitting file data .done +Committing transaction... +Committed revision 8. +$ svn update +Updating '.': +At revision 8. +#+END_SRC + +Checking the log, we see those new svn commits: +#+BEGIN_SRC sh +$ svn log +------------------------------------------------------------------------ +r8 | tony | 2016-06-24 11:08:42 +0200 (Fri, 24 Jun 2016) | 1 line + +Add link to README +------------------------------------------------------------------------ +r7 | tony | 2016-06-24 11:07:04 +0200 (Fri, 24 Jun 2016) | 1 line + +Add a new README +------------------------------------------------------------------------ +r6 | seanius | 2006-01-27 07:38:25 +0100 (Fri, 27 Jan 2006) | 1 line + +fix breakage in rules +#+END_SRC + +Loading the svn repository, we see 2 new swh revisions: +#+BEGIN_SRC sh +b'tree 752c52134dcbf2fff13c7be1ce4e9e5dbf428a59\nparent 4876cb10aec6f708f7466dddf547567b65f6c39c\nauthor tony 1466759224.2817 +0000\ncommitter tony 1466759224.2817 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 7\n\nAdd a new README' +[2016-06-24 11:18:21,055: DEBUG/Worker-1] rev: 7, swhrev: 7f5bc909c29d4e93d8ccfdda516e51ed44930ee1, dir: 752c52134dcbf2fff13c7be1ce4e9e5dbf428a59 +b'tree 39c813fb4717a4864bacefbd90b51a3241ae4140\nparent 7f5bc909c29d4e93d8ccfdda516e51ed44930ee1\nauthor tony 1466759322.099151 +0000\ncommitter tony 1466759322.099151 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 8\n\nAdd link to README' +[2016-06-24 11:18:21,066: DEBUG/Worker-1] rev: 8, swhrev: 38d81702cb28db4f1a6821e64321e5825d1f7fd6, dir: 39c813fb4717a4864bacefbd90b51a3241ae4140 +#+END_SRC + +|--------------+------------------------------------------+------------------------------------------| +| svn revision | swh revision | tree | +|--------------+------------------------------------------+------------------------------------------| +| 7 | 7f5bc909c29d4e93d8ccfdda516e51ed44930ee1 | 752c52134dcbf2fff13c7be1ce4e9e5dbf428a59 | +| 8 | 38d81702cb28db4f1a6821e64321e5825d1f7fd6 | 39c813fb4717a4864bacefbd90b51a3241ae4140 | +|--------------+------------------------------------------+------------------------------------------| +*** Checks +**** Trees + +#+BEGIN_SRC sh +$ pwd +/home/storage/svn/working-copies/pkg-gourmet +$ cd ..; rm -rf pkg-gourmet; svn export --ignore-keywords file:///home/storage/svn/repos/pkg-gourmet@7; cd pkg-gourmet; swh-hashtree --path . +A pkg-gourmet +A pkg-gourmet/foo +A pkg-gourmet/foo/bar +A pkg-gourmet/foo/bar/README +A pkg-gourmet/gourmet +A pkg-gourmet/gourmet/branches +A pkg-gourmet/gourmet/tags +A pkg-gourmet/gourmet/trunk +A pkg-gourmet/gourmet/trunk/debian +A pkg-gourmet/gourmet/trunk/debian/patches +A pkg-gourmet/gourmet/trunk/debian/patches/00list +A pkg-gourmet/gourmet/trunk/debian/patches/01_printer_warning.dpatch +A pkg-gourmet/gourmet/trunk/debian/README.Maintainer +A pkg-gourmet/gourmet/trunk/debian/TODO +A pkg-gourmet/gourmet/trunk/debian/changelog +A pkg-gourmet/gourmet/trunk/debian/compat +A pkg-gourmet/gourmet/trunk/debian/control +A pkg-gourmet/gourmet/trunk/debian/copyright +A pkg-gourmet/gourmet/trunk/debian/dirs +A pkg-gourmet/gourmet/trunk/debian/docs +A pkg-gourmet/gourmet/trunk/debian/gourmet.1 +A pkg-gourmet/gourmet/trunk/debian/menu +A pkg-gourmet/gourmet/trunk/debian/postinst +A pkg-gourmet/gourmet/trunk/debian/postrm +A pkg-gourmet/gourmet/trunk/debian/prerm +A pkg-gourmet/gourmet/trunk/debian/recbox.xpm +A pkg-gourmet/gourmet/trunk/debian/rules +A pkg-gourmet/gourmet/trunk/debian/source.lintian-overrides +Exported revision 7. +752c52134dcbf2fff13c7be1ce4e9e5dbf428a59 +$ cd ..; rm -rf pkg-gourmet; svn export --ignore-keywords file:///home/storage/svn/repos/pkg-gourmet@8; cd pkg-gourmet; swh-hashtree --path . +A pkg-gourmet +A pkg-gourmet/foo +A pkg-gourmet/foo/bar +A pkg-gourmet/foo/bar/README +A pkg-gourmet/gourmet +A pkg-gourmet/gourmet/branches +A pkg-gourmet/gourmet/tags +A pkg-gourmet/gourmet/trunk +A pkg-gourmet/gourmet/trunk/debian +A pkg-gourmet/gourmet/trunk/debian/patches +A pkg-gourmet/gourmet/trunk/debian/patches/00list +A pkg-gourmet/gourmet/trunk/debian/patches/01_printer_warning.dpatch +A pkg-gourmet/gourmet/trunk/debian/README.Maintainer +A pkg-gourmet/gourmet/trunk/debian/TODO +A pkg-gourmet/gourmet/trunk/debian/changelog +A pkg-gourmet/gourmet/trunk/debian/compat +A pkg-gourmet/gourmet/trunk/debian/control +A pkg-gourmet/gourmet/trunk/debian/copyright +A pkg-gourmet/gourmet/trunk/debian/dirs +A pkg-gourmet/gourmet/trunk/debian/docs +A pkg-gourmet/gourmet/trunk/debian/gourmet.1 +A pkg-gourmet/gourmet/trunk/debian/menu +A pkg-gourmet/gourmet/trunk/debian/postinst +A pkg-gourmet/gourmet/trunk/debian/postrm +A pkg-gourmet/gourmet/trunk/debian/prerm +A pkg-gourmet/gourmet/trunk/debian/recbox.xpm +A pkg-gourmet/gourmet/trunk/debian/rules +A pkg-gourmet/gourmet/trunk/debian/source.lintian-overrides +A pkg-gourmet/README +Exported revision 8. +39c813fb4717a4864bacefbd90b51a3241ae4140 +#+END_SRC + +Trees ok! + +**** Revisions + +#+BEGIN_SRC sh +$ git-revhash 'tree 752c52134dcbf2fff13c7be1ce4e9e5dbf428a59\nparent 4876cb10aec6f708f7466dddf547567b65f6c39c\nauthor tony 1466759224.2817 +0000\ncommitter tony 1466759224.2817 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 7\n\nAdd a new README' +7f5bc909c29d4e93d8ccfdda516e51ed44930ee1 +$ git-revhash 'tree 39c813fb4717a4864bacefbd90b51a3241ae4140\nparent 7f5bc909c29d4e93d8ccfdda516e51ed44930ee1\nauthor tony 1466759322.099151 +0000\ncommitter tony 1466759322.099151 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 8\n\nAdd link to README +38d81702cb28db4f1a6821e64321e5825d1f7fd6 +#+END_SRC diff --git a/swh/loader/svn/tests/test_loader.py b/swh/loader/svn/tests/test_loader.py index 0b73d71..d4208d8 100644 --- a/swh/loader/svn/tests/test_loader.py +++ b/swh/loader/svn/tests/test_loader.py @@ -1,268 +1,390 @@ # Copyright (C) 2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import shutil import subprocess import tempfile import unittest from nose.tools import istest from swh.core import hashutil from swh.loader.svn.loader import GitSvnSvnLoader, SWHSvnLoader # Define loaders with no storage # They'll just accumulate the data in place # Only for testing purposes. class TestSvnLoader: """Mixin class to inhibit the persistence and keep in memory the data sent for storage. cf. GitSvnLoaderNoStorage, SWHSvnLoaderNoStorage """ def __init__(self, svn_url, destination_path, origin): super().__init__(svn_url, destination_path, origin) # We don't want to persist any result in this test context self.config['send_contents'] = False self.config['send_directories'] = False self.config['send_revisions'] = False self.config['send_releases'] = False self.config['send_occurrences'] = False # Init the state self.all_contents = [] self.all_directories = [] self.all_revisions = [] self.all_releases = [] self.all_occurrences = [] def maybe_load_contents(self, all_contents): self.all_contents.extend(all_contents) def maybe_load_directories(self, all_directories): self.all_directories.extend(all_directories) def maybe_load_revisions(self, all_revisions): self.all_revisions.extend(all_revisions) def maybe_load_releases(self, releases): raise ValueError('If called, the test must break.') def maybe_load_occurrences(self, all_occurrences): self.all_occurrences.extend(all_occurrences) class GitSvnLoaderNoStorage(TestSvnLoader, GitSvnSvnLoader): """A GitSvnLoader with no persistence. - Context: Load an svn repository using the git-svn policy. + Context: + Load an svn repository using the git-svn policy. """ def __init__(self, svn_url, destination_path, origin): super().__init__(svn_url, destination_path, origin) class SWHSvnLoaderNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: - Load a new svn repository using the swh policy (so no update). + Load a new svn repository using the swh policy (so no update). """ def __init__(self, svn_url, destination_path, origin): super().__init__(svn_url, destination_path, origin) def swh_previous_revision(self): """We do not know this repository so no revision. """ return None class SWHSvnLoaderUpdateNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: - Load a known svn repository using the swh policy so we need to update it. + Load a known svn repository using the swh policy. + We can either: + - do nothing since it does not contain any new commit (so no + change) + - either check its history is not altered and update in + consequence by loading the new revision """ def __init__(self, svn_url, destination_path, origin): super().__init__(svn_url, destination_path, origin) def swh_previous_revision(self): """Avoid the storage persistence call and return the expected previous revision for that repository. Check the following for explanation about the hashes: - test_loader.org for (swh policy). - cf. SWHSvnLoaderITTest """ return { 'id': hashutil.hex_to_hash( '4876cb10aec6f708f7466dddf547567b65f6c39c'), 'parents': [hashutil.hex_to_hash( 'a3a577948fdbda9d1061913b77a1588695eadb41')], 'directory': hashutil.hex_to_hash( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ - # should be the right uuid but we don't care much here - ['svn_repo_uuid', ''], + ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], + ['svn_revision', b'6'] + ] + } + } + + +class SWHSvnLoaderUpdateHistoryAlteredNoStorage(TestSvnLoader, SWHSvnLoader): + """An SWHSVNLoader with no persistence. + + Context: Load a known svn repository using the swh policy with its + history altered so we do not update it. + + """ + def __init__(self, svn_url, destination_path, origin): + super().__init__(svn_url, destination_path, origin) + + def swh_previous_revision(self): + """Avoid the storage persistence call and return the expected previous + revision for that repository. + + Check the following for explanation about the hashes: + - test_loader.org for (swh policy). + - cf. SWHSvnLoaderITTest + + """ + return { + # Changed the revision id's hash to simulate history altered + 'id': hashutil.hex_to_hash( + 'badbadbadbadf708f7466dddf547567b65f6c39d'), + 'parents': [hashutil.hex_to_hash( + 'a3a577948fdbda9d1061913b77a1588695eadb41')], + 'directory': hashutil.hex_to_hash( + '0deab3023ac59398ae467fc4bff5583008af1ee2'), + 'target_type': 'revision', + 'metadata': { + 'extra_headers': [ + ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', b'6'] ] } } class BaseTestLoader(unittest.TestCase): - def setUp(self, filename='pkg-gourmet'): + """Base test loader class. + + In its setup, it's uncompressing a local svn mirror to /tmp. + + """ + def setUp(self, archive_name='pkg-gourmet.tgz', filename='pkg-gourmet'): self.tmp_root_path = tempfile.mkdtemp() start_path = os.path.dirname(__file__) svn_mirror_repo = os.path.join(start_path, '../../../../..', 'swh-storage-testdata', 'svn-folders', - filename + '.tgz') + archive_name) # uncompress the sample folder subprocess.check_output( ['tar', 'xvf', svn_mirror_repo, '-C', self.tmp_root_path], ) self.svn_mirror_url = 'file://' + self.tmp_root_path + '/' + filename self.destination_path = os.path.join( self.tmp_root_path, 'working-copy') def tearDown(self): - super().tearDownClass() - shutil.rmtree(self.tmp_root_path) class GitSvnLoaderITTest(BaseTestLoader): def setUp(self): super().setUp() self.origin = {'id': 1, 'type': 'svn', 'url': 'file:///dev/null'} self.loader = GitSvnLoaderNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """Process a repository with gitsvn policy should be ok.""" # when self.loader.process_repository() # then self.assertEquals(len(self.loader.all_revisions), 6) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 1) last_revision = 'bad4a83737f337d47e0ba681478214b07a707218' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash # noqa '22c0fa5195a53f2e733ec75a9b6e9d1624a8b771': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', # noqa '17a631d474f49bbebfdf3d885dcde470d7faafd7': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', # noqa 'c8a9172b2a615d461154f61158180de53edc6070': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', # noqa '7c8f83394b6e8966eb46f0d3416c717612198a4b': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', # noqa '852547b3b2bb76c8582cee963e8aa180d552a15c': 'ab047e38d1532f61ff5c3621202afc3e763e9945', # noqa last_revision: '9bcfc25001b71c333b4b5a89224217de81c56e2e', # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) occ = self.loader.all_occurrences[0] self.assertEquals(hashutil.hash_to_hex(occ['target']), last_revision) self.assertEquals(occ['origin'], self.origin['id']) class SWHSvnLoaderNewRepositoryITTest(BaseTestLoader): def setUp(self): super().setUp() self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.loader = SWHSvnLoaderNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """Process a new repository with swh policy should be ok. """ # when self.loader.process_repository() # then self.assertEquals(len(self.loader.all_revisions), 6) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 1) last_revision = '4876cb10aec6f708f7466dddf547567b65f6c39c' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71': '669a71cce6c424a81ba42b7dc5d560d32252f0ca', # noqa '95edacc8848369d6fb1608e887d6d2474fd5224f': '008ac97a1118560797c50e3392fa1443acdaa349', # noqa 'fef26ea45a520071711ba2b9d16a2985ee837021': '3780effbe846a26751a95a8c95c511fb72be15b4', # noqa '3f51abf3b3d466571be0855dfa67e094f9ceff1b': 'ffcca9b09c5827a6b8137322d4339c8055c3ee1e', # noqa 'a3a577948fdbda9d1061913b77a1588695eadb41': '7dc52cc04c3b8bd7c085900d60c159f7b846f866', # noqa last_revision: '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) occ = self.loader.all_occurrences[0] self.assertEquals(hashutil.hash_to_hex(occ['target']), last_revision) self.assertEquals(occ['origin'], self.origin['id']) class SWHSvnLoaderUpdateWithNoChangeITTest(BaseTestLoader): def setUp(self): super().setUp() self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.loader = SWHSvnLoaderUpdateNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """Process a known repository with swh policy and no new data should be ok. """ # when self.loader.process_repository() # then self.assertEquals(len(self.loader.all_revisions), 0) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 0) + + +class SWHSvnLoaderUpdateWithHistoryAlteredITTest(BaseTestLoader): + def setUp(self): + # the svn repository pkg-gourmet has been updated with changes + super().setUp(archive_name='pkg-gourmet-with-updates.tgz') + + self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} + + self.loader = SWHSvnLoaderUpdateHistoryAlteredNoStorage( + svn_url=self.svn_mirror_url, + destination_path=self.destination_path, + origin=self.origin) + + @istest + def process_repository(self): + """Process a known repository with swh policy and history altered should + stop and do nothing. + + """ + # when + self.loader.process_repository() + + # then + # we got the previous run's last revision (rev 6) + # so 2 news + 1 old + self.assertEquals(len(self.loader.all_revisions), 0) + self.assertEquals(len(self.loader.all_releases), 0) + self.assertEquals(len(self.loader.all_occurrences), 0) + + +class SWHSvnLoaderUpdateWithChangesITTest(BaseTestLoader): + def setUp(self): + # the svn repository pkg-gourmet has been updated with changes + super().setUp(archive_name='pkg-gourmet-with-updates.tgz') + + self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} + + self.loader = SWHSvnLoaderUpdateNoStorage( + svn_url=self.svn_mirror_url, + destination_path=self.destination_path, + origin=self.origin) + + @istest + def process_repository(self): + """Process a known repository with swh policy and new data should + yield new revisions and occurrence. + + """ + # when + self.loader.process_repository() + + # then + # we got the previous run's last revision (rev 6) + # so 2 news + 1 old + self.assertEquals(len(self.loader.all_revisions), 2) + self.assertEquals(len(self.loader.all_releases), 0) + self.assertEquals(len(self.loader.all_occurrences), 1) + + last_revision = '38d81702cb28db4f1a6821e64321e5825d1f7fd6' + # cf. test_loader.org for explaining from where those hashes + # come from + expected_revisions = { + # revision hash | directory hash + '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa + last_revision: '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa + } + + for rev in self.loader.all_revisions: + rev_id = hashutil.hash_to_hex(rev['id']) + directory_id = hashutil.hash_to_hex(rev['directory']) + + self.assertEquals(expected_revisions[rev_id], directory_id) + + occ = self.loader.all_occurrences[0] + self.assertEquals(hashutil.hash_to_hex(occ['target']), last_revision) + self.assertEquals(occ['origin'], self.origin['id'])