diff --git a/swh/loader/svn/tests/test_loader.org b/swh/loader/svn/tests/test_loader.org index 5537487..44fb5be 100644 --- a/swh/loader/svn/tests/test_loader.org +++ b/swh/loader/svn/tests/test_loader.org @@ -1,284 +1,301 @@ #+title: Prepare test_converters.py it tests #+author: ardumont * Requisite: #+BEGIN_SRC sh sudo apt install subversion git-svn #+END_SRC * Create mirror repository Then: #+BEGIN_SRC sh ./init-svn-repository.sh /home/storage/svn/repos/pkg-gourmet svn://svn.debian.org/svn/pkg-gourmet/ #+END_SRC Note: Saved as ../../../../bin/init-svn-repository.sh And now we have a mirror svn repository at file:///home/storage/svn/repos/pkg-gourmet * git-svn policy `git svn clone` the repository and parse the git log entries for the needed data. #+BEGIN_SRC sh git svn clone file:///home/storage/svn/repos/pkg-gourmet -q --no-metadata cd pkg-gourmet # commit git log --format=raw --reverse | grep '^commit ' | awk '{print $2}' # tree git log --format=raw --reverse | grep '^tree ' | awk '{print $2}' #+END_SRC Those are the data to check when done parsing the repository: |------------------------------------------+------------------------------------------| | revision | tree | |------------------------------------------+------------------------------------------| | 22c0fa5195a53f2e733ec75a9b6e9d1624a8b771 | 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | | 17a631d474f49bbebfdf3d885dcde470d7faafd7 | 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | | c8a9172b2a615d461154f61158180de53edc6070 | 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | | 7c8f83394b6e8966eb46f0d3416c717612198a4b | 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | | 852547b3b2bb76c8582cee963e8aa180d552a15c | ab047e38d1532f61ff5c3621202afc3e763e9945 | | bad4a83737f337d47e0ba681478214b07a707218 | 9bcfc25001b71c333b4b5a89224217de81c56e2e | |------------------------------------------+------------------------------------------| * swh policy ** New repository For this one this was more tedious. #+BEGIN_SRC sh $ svn export --ignore-keywords file:///home/storage/svn/repos/pkg-gourmet@1 #+END_SRC The export does not expand the keywords and does not include the .svn folder. Then: #+BEGIN_SRC sh $ cd pkg-gourmet $ swh-hashtree --path . 669a71cce6c424a81ba42b7dc5d560d32252f0ca #+END_SRC Note: ../../../../bin/hashtree Then for the next revision: #+BEGIN_SRC sh cd .. ; rm -rf pkg-gourmet; svn export --ignore-keywords file:///home/storage/svn/repos/pkg-gourmet@2 A pkg-gourmet A pkg-gourmet/gourmet A pkg-gourmet/gourmet/trunk Exported revision 2. $ cd pkg-gourmet && swh-hashtree --path . 008ac97a1118560797c50e3392fa1443acdaa349 #+END_SRC etc... |--------------+------------------------------------------+------------------------------------------| | svn revision | swh revision | tree | |--------------+------------------------------------------+------------------------------------------| | 1 | 0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71 | 669a71cce6c424a81ba42b7dc5d560d32252f0ca | | 2 | 95edacc8848369d6fb1608e887d6d2474fd5224f | 008ac97a1118560797c50e3392fa1443acdaa349 | | 3 | fef26ea45a520071711ba2b9d16a2985ee837021 | 3780effbe846a26751a95a8c95c511fb72be15b4 | | 4 | 3f51abf3b3d466571be0855dfa67e094f9ceff1b | ffcca9b09c5827a6b8137322d4339c8055c3ee1e | | 5 | a3a577948fdbda9d1061913b77a1588695eadb41 | 7dc52cc04c3b8bd7c085900d60c159f7b846f866 | | 6 | 4876cb10aec6f708f7466dddf547567b65f6c39c | 0deab3023ac59398ae467fc4bff5583008af1ee2 | |--------------+------------------------------------------+------------------------------------------| For the revision, cheating a little. That is adapting swh.model.identifiers.revision_identifiers to print the commit's manifest: #+BEGIN_SRC sh b'tree 669a71cce6c424a81ba42b7dc5d560d32252f0ca\nauthor seanius 1138341038.645397 +0000\ncommitter seanius 1138341038.645397 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 1\n\nmaking dir structure...' [2016-06-23 12:35:39,291: DEBUG/Worker-1] rev: 1, swhrev: 0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71, dir: 669a71cce6c424a81ba42b7dc5d560d32252f0ca b'tree 008ac97a1118560797c50e3392fa1443acdaa349\nparent 0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71\nauthor seanius 1138341044.821526 +0000\ncommitter seanius 1138341044.821526 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 2\n\nmaking dir structure...' [2016-06-23 12:35:39,302: DEBUG/Worker-1] rev: 2, swhrev: 95edacc8848369d6fb1608e887d6d2474fd5224f, dir: 008ac97a1118560797c50e3392fa1443acdaa349 b'tree 3780effbe846a26751a95a8c95c511fb72be15b4\nparent 95edacc8848369d6fb1608e887d6d2474fd5224f\nauthor seanius 1138341057.282488 +0000\ncommitter seanius 1138341057.282488 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 3\n\nmaking dir structure...' [2016-06-23 12:35:39,313: DEBUG/Worker-1] rev: 3, swhrev: fef26ea45a520071711ba2b9d16a2985ee837021, dir: 3780effbe846a26751a95a8c95c511fb72be15b4 b'tree ffcca9b09c5827a6b8137322d4339c8055c3ee1e\nparent fef26ea45a520071711ba2b9d16a2985ee837021\nauthor seanius 1138341064.191867 +0000\ncommitter seanius 1138341064.191867 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 4\n\nmaking dir structure...' [2016-06-23 12:35:39,322: DEBUG/Worker-1] rev: 4, swhrev: 3f51abf3b3d466571be0855dfa67e094f9ceff1b, dir: ffcca9b09c5827a6b8137322d4339c8055c3ee1e b'tree 7dc52cc04c3b8bd7c085900d60c159f7b846f866\nparent 3f51abf3b3d466571be0855dfa67e094f9ceff1b\nauthor seanius 1138342632.066765 +0000\ncommitter seanius 1138342632.066765 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 5\n\ninitial import' [2016-06-23 12:35:39,339: DEBUG/Worker-1] rev: 5, swhrev: a3a577948fdbda9d1061913b77a1588695eadb41, dir: 7dc52cc04c3b8bd7c085900d60c159f7b846f866 b'tree 0deab3023ac59398ae467fc4bff5583008af1ee2\nparent a3a577948fdbda9d1061913b77a1588695eadb41\nauthor seanius 1138343905.448277 +0000\ncommitter seanius 1138343905.448277 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 6\n\nfix breakage in rules' [2016-06-23 12:35:39,348: DEBUG/Worker-1] rev: 6, swhrev: 4876cb10aec6f708f7466dddf547567b65f6c39c, dir: 0deab3023ac59398ae467fc4bff5583008af1ee2 [2016-06-23 12:35:39,355: INFO/Worker-1] Processed 6 revisions: [0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71, ...] #+END_SRC Then checking the manifest's hash is ok: #+BEGIN_SRC sh $ echo -en 'tree 669a71cce6c424a81ba42b7dc5d560d32252f0ca\nauthor seanius 1138341038.645397 +0000\ncommitter seanius 1138341038.645397 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 1\n\nmaking dir structure...' | git hash-object -t commit --stdin 0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71 #+END_SRC And all is ok. ** Update existing repository Checkout on disk the repository and do some modifications on it: #+BEGIN_SRC sh $ svn co file:///home/storage/svn/repos/pkg-gourmet/ A pkg-gourmet/gourmet A pkg-gourmet/gourmet/branches A pkg-gourmet/gourmet/tags A pkg-gourmet/gourmet/trunk A pkg-gourmet/gourmet/trunk/debian A pkg-gourmet/gourmet/trunk/debian/patches A pkg-gourmet/gourmet/trunk/debian/patches/00list A pkg-gourmet/gourmet/trunk/debian/patches/01_printer_warning.dpatch A pkg-gourmet/gourmet/trunk/debian/README.Maintainer A pkg-gourmet/gourmet/trunk/debian/TODO A pkg-gourmet/gourmet/trunk/debian/changelog A pkg-gourmet/gourmet/trunk/debian/compat A pkg-gourmet/gourmet/trunk/debian/control A pkg-gourmet/gourmet/trunk/debian/copyright A pkg-gourmet/gourmet/trunk/debian/dirs A pkg-gourmet/gourmet/trunk/debian/docs A pkg-gourmet/gourmet/trunk/debian/gourmet.1 A pkg-gourmet/gourmet/trunk/debian/menu A pkg-gourmet/gourmet/trunk/debian/postinst A pkg-gourmet/gourmet/trunk/debian/postrm A pkg-gourmet/gourmet/trunk/debian/prerm A pkg-gourmet/gourmet/trunk/debian/recbox.xpm A pkg-gourmet/gourmet/trunk/debian/rules A pkg-gourmet/gourmet/trunk/debian/source.lintian-overrides Checked out revision 6. $ cd pkg-gourmet $ mkdir foo/bar/ -p $ em foo/bar/new-file % svn add foo A foo A foo/bar A foo/bar/README $ svn commit -m 'Add a new README' Adding foo Adding foo/bar Adding foo/bar/README Transmitting file data .done Committing transaction... Committed revision 7. $ ln -s foo/bar/README README $ svn add README A README $ svn commit -m 'Add link to README' Adding README Transmitting file data .done Committing transaction... Committed revision 8. $ svn update Updating '.': At revision 8. #+END_SRC Checking the log, we see those new svn commits: #+BEGIN_SRC sh $ svn log +------------------------------------------------------------------------ +r11 | tony | 2016-07-11 10:38:43 +0200 (Mon, 11 Jul 2016) | 1 line + +Delete broken link +------------------------------------------------------------------------ +r10 | tony | 2016-07-11 10:09:13 +0200 (Mon, 11 Jul 2016) | 1 line + +Add bin/hello executable +------------------------------------------------------------------------ +r9 | tony | 2016-07-07 11:13:52 +0200 (Thu, 07 Jul 2016) | 1 line + ------------------------------------------------------------------------ r8 | tony | 2016-06-24 11:08:42 +0200 (Fri, 24 Jun 2016) | 1 line Add link to README ------------------------------------------------------------------------ r7 | tony | 2016-06-24 11:07:04 +0200 (Fri, 24 Jun 2016) | 1 line Add a new README ------------------------------------------------------------------------ r6 | seanius | 2006-01-27 07:38:25 +0100 (Fri, 27 Jan 2006) | 1 line fix breakage in rules #+END_SRC Loading the svn repository, we see 2 new swh revisions: #+BEGIN_SRC sh b'tree 752c52134dcbf2fff13c7be1ce4e9e5dbf428a59\nparent 4876cb10aec6f708f7466dddf547567b65f6c39c\nauthor tony 1466759224.2817 +0000\ncommitter tony 1466759224.2817 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 7\n\nAdd a new README' [2016-06-24 11:18:21,055: DEBUG/Worker-1] rev: 7, swhrev: 7f5bc909c29d4e93d8ccfdda516e51ed44930ee1, dir: 752c52134dcbf2fff13c7be1ce4e9e5dbf428a59 b'tree 39c813fb4717a4864bacefbd90b51a3241ae4140\nparent 7f5bc909c29d4e93d8ccfdda516e51ed44930ee1\nauthor tony 1466759322.099151 +0000\ncommitter tony 1466759322.099151 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 8\n\nAdd link to README' [2016-06-24 11:18:21,066: DEBUG/Worker-1] rev: 8, swhrev: 38d81702cb28db4f1a6821e64321e5825d1f7fd6, dir: 39c813fb4717a4864bacefbd90b51a3241ae4140 +[2016-07-11 10:47:18,609: DEBUG/Worker-1] rev: 9, swhrev: 99c27ebbd43feca179ac0e895af131d8314cafe1, dir: 3397ca7f709639cbd36b18a0d1b70bce80018c45 +[2016-07-11 10:47:18,644: DEBUG/Worker-1] rev: 10, swhrev: 902f29b4323a9b9de3af6d28e72dd581e76d9397, dir: c4e12483f0a13e6851459295a4ae735eb4e4b5c4 +[2016-07-11 10:47:18,831: DEBUG/Worker-1] rev: 11, swhrev: 171dc35522bfd17dda4e90a542a0377fb2fc707a, dir: fd24a76c87a3207428e06612b49860fc78e9f6dc #+END_SRC |--------------+------------------------------------------+------------------------------------------| | svn revision | swh revision | tree | |--------------+------------------------------------------+------------------------------------------| | 7 | 7f5bc909c29d4e93d8ccfdda516e51ed44930ee1 | 752c52134dcbf2fff13c7be1ce4e9e5dbf428a59 | | 8 | 38d81702cb28db4f1a6821e64321e5825d1f7fd6 | 39c813fb4717a4864bacefbd90b51a3241ae4140 | +| 9 | 99c27ebbd43feca179ac0e895af131d8314cafe1 | 3397ca7f709639cbd36b18a0d1b70bce80018c45 | +| 10 | 902f29b4323a9b9de3af6d28e72dd581e76d9397 | c4e12483f0a13e6851459295a4ae735eb4e4b5c4 | +| 11 | 171dc35522bfd17dda4e90a542a0377fb2fc707a | fd24a76c87a3207428e06612b49860fc78e9f6dc | |--------------+------------------------------------------+------------------------------------------| *** Checks **** Trees #+BEGIN_SRC sh $ pwd /home/storage/svn/working-copies/pkg-gourmet $ cd ..; rm -rf pkg-gourmet; svn export --ignore-keywords file:///home/storage/svn/repos/pkg-gourmet@7; cd pkg-gourmet; swh-hashtree --path . A pkg-gourmet A pkg-gourmet/foo A pkg-gourmet/foo/bar A pkg-gourmet/foo/bar/README A pkg-gourmet/gourmet A pkg-gourmet/gourmet/branches A pkg-gourmet/gourmet/tags A pkg-gourmet/gourmet/trunk A pkg-gourmet/gourmet/trunk/debian A pkg-gourmet/gourmet/trunk/debian/patches A pkg-gourmet/gourmet/trunk/debian/patches/00list A pkg-gourmet/gourmet/trunk/debian/patches/01_printer_warning.dpatch A pkg-gourmet/gourmet/trunk/debian/README.Maintainer A pkg-gourmet/gourmet/trunk/debian/TODO A pkg-gourmet/gourmet/trunk/debian/changelog A pkg-gourmet/gourmet/trunk/debian/compat A pkg-gourmet/gourmet/trunk/debian/control A pkg-gourmet/gourmet/trunk/debian/copyright A pkg-gourmet/gourmet/trunk/debian/dirs A pkg-gourmet/gourmet/trunk/debian/docs A pkg-gourmet/gourmet/trunk/debian/gourmet.1 A pkg-gourmet/gourmet/trunk/debian/menu A pkg-gourmet/gourmet/trunk/debian/postinst A pkg-gourmet/gourmet/trunk/debian/postrm A pkg-gourmet/gourmet/trunk/debian/prerm A pkg-gourmet/gourmet/trunk/debian/recbox.xpm A pkg-gourmet/gourmet/trunk/debian/rules A pkg-gourmet/gourmet/trunk/debian/source.lintian-overrides Exported revision 7. 752c52134dcbf2fff13c7be1ce4e9e5dbf428a59 $ cd ..; rm -rf pkg-gourmet; svn export --ignore-keywords file:///home/storage/svn/repos/pkg-gourmet@8; cd pkg-gourmet; swh-hashtree --path . A pkg-gourmet A pkg-gourmet/foo A pkg-gourmet/foo/bar A pkg-gourmet/foo/bar/README A pkg-gourmet/gourmet A pkg-gourmet/gourmet/branches A pkg-gourmet/gourmet/tags A pkg-gourmet/gourmet/trunk A pkg-gourmet/gourmet/trunk/debian A pkg-gourmet/gourmet/trunk/debian/patches A pkg-gourmet/gourmet/trunk/debian/patches/00list A pkg-gourmet/gourmet/trunk/debian/patches/01_printer_warning.dpatch A pkg-gourmet/gourmet/trunk/debian/README.Maintainer A pkg-gourmet/gourmet/trunk/debian/TODO A pkg-gourmet/gourmet/trunk/debian/changelog A pkg-gourmet/gourmet/trunk/debian/compat A pkg-gourmet/gourmet/trunk/debian/control A pkg-gourmet/gourmet/trunk/debian/copyright A pkg-gourmet/gourmet/trunk/debian/dirs A pkg-gourmet/gourmet/trunk/debian/docs A pkg-gourmet/gourmet/trunk/debian/gourmet.1 A pkg-gourmet/gourmet/trunk/debian/menu A pkg-gourmet/gourmet/trunk/debian/postinst A pkg-gourmet/gourmet/trunk/debian/postrm A pkg-gourmet/gourmet/trunk/debian/prerm A pkg-gourmet/gourmet/trunk/debian/recbox.xpm A pkg-gourmet/gourmet/trunk/debian/rules A pkg-gourmet/gourmet/trunk/debian/source.lintian-overrides A pkg-gourmet/README Exported revision 8. 39c813fb4717a4864bacefbd90b51a3241ae4140 #+END_SRC Trees ok! **** Revisions #+BEGIN_SRC sh $ git-revhash 'tree 752c52134dcbf2fff13c7be1ce4e9e5dbf428a59\nparent 4876cb10aec6f708f7466dddf547567b65f6c39c\nauthor tony 1466759224.2817 +0000\ncommitter tony 1466759224.2817 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 7\n\nAdd a new README' 7f5bc909c29d4e93d8ccfdda516e51ed44930ee1 $ git-revhash 'tree 39c813fb4717a4864bacefbd90b51a3241ae4140\nparent 7f5bc909c29d4e93d8ccfdda516e51ed44930ee1\nauthor tony 1466759322.099151 +0000\ncommitter tony 1466759322.099151 +0000\nsvn_repo_uuid 3187e211-bb14-4c82-9596-0b59d67cd7f4\nsvn_revision 8\n\nAdd link to README 38d81702cb28db4f1a6821e64321e5825d1f7fd6 #+END_SRC diff --git a/swh/loader/svn/tests/test_loader.py b/swh/loader/svn/tests/test_loader.py index cfe244b..92aae28 100644 --- a/swh/loader/svn/tests/test_loader.py +++ b/swh/loader/svn/tests/test_loader.py @@ -1,612 +1,620 @@ # Copyright (C) 2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import shutil import subprocess import tempfile import unittest from nose.tools import istest from swh.core import hashutil from swh.loader.svn.loader import GitSvnSvnLoader, SWHSvnLoader # Define loaders with no storage # They'll just accumulate the data in place # Only for testing purposes. class TestSvnLoader: """Mixin class to inhibit the persistence and keep in memory the data sent for storage. cf. GitSvnLoaderNoStorage, SWHSvnLoaderNoStorage """ def __init__(self, svn_url, destination_path, origin): super().__init__(svn_url, destination_path, origin) # We don't want to persist any result in this test context self.config['send_contents'] = False self.config['send_directories'] = False self.config['send_revisions'] = False self.config['send_releases'] = False self.config['send_occurrences'] = False # Init the state self.all_contents = [] self.all_directories = [] self.all_revisions = [] self.all_releases = [] self.all_occurrences = [] def maybe_load_contents(self, all_contents): self.all_contents.extend(all_contents) def maybe_load_directories(self, all_directories): self.all_directories.extend(all_directories) def maybe_load_revisions(self, all_revisions): self.all_revisions.extend(all_revisions) def maybe_load_releases(self, releases): raise ValueError('If called, the test must break.') def maybe_load_occurrences(self, all_occurrences): self.all_occurrences.extend(all_occurrences) class GitSvnLoaderNoStorage(TestSvnLoader, GitSvnSvnLoader): """A GitSvnLoader with no persistence. Context: Load an svn repository using the git-svn policy. """ def __init__(self, svn_url, destination_path, origin): super().__init__(svn_url, destination_path, origin) class SWHSvnLoaderNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: Load a new svn repository using the swh policy (so no update). """ def swh_previous_revision(self): """We do not know this repository so no revision. """ return None class SWHSvnLoaderUpdateNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: Load a known svn repository using the swh policy. We can either: - do nothing since it does not contain any new commit (so no change) - either check its history is not altered and update in consequence by loading the new revision """ def swh_previous_revision(self): """Avoid the storage persistence call and return the expected previous revision for that repository. Check the following for explanation about the hashes: - test_loader.org for (swh policy). - cf. SWHSvnLoaderITTest """ return { 'id': hashutil.hex_to_hash( '4876cb10aec6f708f7466dddf547567b65f6c39c'), 'parents': [hashutil.hex_to_hash( 'a3a577948fdbda9d1061913b77a1588695eadb41')], 'directory': hashutil.hex_to_hash( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '6'] ] } } class SWHSvnLoaderUpdateHistoryAlteredNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: Load a known svn repository using the swh policy with its history altered so we do not update it. """ def swh_previous_revision(self): """Avoid the storage persistence call and return the expected previous revision for that repository. Check the following for explanation about the hashes: - test_loader.org for (swh policy). - cf. SWHSvnLoaderITTest """ return { # Changed the revision id's hash to simulate history altered 'id': hashutil.hex_to_hash( 'badbadbadbadf708f7466dddf547567b65f6c39d'), 'parents': [hashutil.hex_to_hash( 'a3a577948fdbda9d1061913b77a1588695eadb41')], 'directory': hashutil.hex_to_hash( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', b'6'] ] } } class BaseTestLoader(unittest.TestCase): """Base test loader class. In its setup, it's uncompressing a local svn mirror to /tmp. """ def setUp(self, archive_name='pkg-gourmet.tgz', filename='pkg-gourmet'): self.tmp_root_path = tempfile.mkdtemp() start_path = os.path.dirname(__file__) svn_mirror_repo = os.path.join(start_path, '../../../../..', 'swh-storage-testdata', 'svn-folders', archive_name) # uncompress the sample folder subprocess.check_output( ['tar', 'xvf', svn_mirror_repo, '-C', self.tmp_root_path], ) self.svn_mirror_url = 'file://' + self.tmp_root_path + '/' + filename self.destination_path = os.path.join( self.tmp_root_path, 'working-copy') def tearDown(self): shutil.rmtree(self.tmp_root_path) class GitSvnLoaderITTest(BaseTestLoader): def setUp(self): super().setUp() self.origin = {'id': 1, 'type': 'svn', 'url': 'file:///dev/null'} self.loader = GitSvnLoaderNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """Process a repository with gitsvn policy should be ok.""" # when self.loader.process_repository() # then self.assertEquals(len(self.loader.all_revisions), 6) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 1) last_revision = 'bad4a83737f337d47e0ba681478214b07a707218' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash # noqa '22c0fa5195a53f2e733ec75a9b6e9d1624a8b771': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', # noqa '17a631d474f49bbebfdf3d885dcde470d7faafd7': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', # noqa 'c8a9172b2a615d461154f61158180de53edc6070': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', # noqa '7c8f83394b6e8966eb46f0d3416c717612198a4b': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', # noqa '852547b3b2bb76c8582cee963e8aa180d552a15c': 'ab047e38d1532f61ff5c3621202afc3e763e9945', # noqa last_revision: '9bcfc25001b71c333b4b5a89224217de81c56e2e', # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) occ = self.loader.all_occurrences[0] self.assertEquals(hashutil.hash_to_hex(occ['target']), last_revision) self.assertEquals(occ['origin'], self.origin['id']) class SWHSvnLoaderNewRepositoryITTest(BaseTestLoader): def setUp(self): super().setUp() self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.loader = SWHSvnLoaderNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """Process a new repository with swh policy should be ok. """ # when self.loader.process_repository() # then self.assertEquals(len(self.loader.all_revisions), 6) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 1) last_revision = '4876cb10aec6f708f7466dddf547567b65f6c39c' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71': '669a71cce6c424a81ba42b7dc5d560d32252f0ca', # noqa '95edacc8848369d6fb1608e887d6d2474fd5224f': '008ac97a1118560797c50e3392fa1443acdaa349', # noqa 'fef26ea45a520071711ba2b9d16a2985ee837021': '3780effbe846a26751a95a8c95c511fb72be15b4', # noqa '3f51abf3b3d466571be0855dfa67e094f9ceff1b': 'ffcca9b09c5827a6b8137322d4339c8055c3ee1e', # noqa 'a3a577948fdbda9d1061913b77a1588695eadb41': '7dc52cc04c3b8bd7c085900d60c159f7b846f866', # noqa last_revision: '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) occ = self.loader.all_occurrences[0] self.assertEquals(hashutil.hash_to_hex(occ['target']), last_revision) self.assertEquals(occ['origin'], self.origin['id']) class SWHSvnLoaderUpdateWithNoChangeITTest(BaseTestLoader): def setUp(self): super().setUp() self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.loader = SWHSvnLoaderUpdateNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """Process a known repository with swh policy and no new data should be ok. """ # when self.loader.process_repository() # then self.assertEquals(len(self.loader.all_revisions), 0) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 0) class SWHSvnLoaderUpdateWithHistoryAlteredITTest(BaseTestLoader): def setUp(self): # the svn repository pkg-gourmet has been updated with changes super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.loader = SWHSvnLoaderUpdateHistoryAlteredNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """Process a known repository with swh policy and history altered should stop and do nothing. """ # when self.loader.process_repository() # then # we got the previous run's last revision (rev 6) # so 2 news + 1 old self.assertEquals(len(self.loader.all_revisions), 0) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 0) class SWHSvnLoaderUpdateWithChangesITTest(BaseTestLoader): def setUp(self): # the svn repository pkg-gourmet has been updated with changes super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.loader = SWHSvnLoaderUpdateNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """Process a known repository with swh policy and new data should yield new revisions and occurrence. """ # when self.loader.process_repository() # then # we got the previous run's last revision (rev 6) # so 2 new - self.assertEquals(len(self.loader.all_revisions), 3) + self.assertEquals(len(self.loader.all_revisions), 5) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 1) - last_revision = '99c27ebbd43feca179ac0e895af131d8314cafe1' + last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa - last_revision: '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa + '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa + '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa + last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) occ = self.loader.all_occurrences[0] self.assertEquals(hashutil.hash_to_hex(occ['target']), last_revision) self.assertEquals(occ['origin'], self.origin['id']) class SWHSvnLoaderUpdateWithUnfinishedLoadingChangesITTest(BaseTestLoader): def setUp(self): super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.loader = SWHSvnLoaderNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """Process a known repository with swh policy, the previous run did not finish, so this finishes the loading """ previous_unfinished_revision = { 'id': hashutil.hex_to_hash( '4876cb10aec6f708f7466dddf547567b65f6c39c'), 'parents': [hashutil.hex_to_hash( 'a3a577948fdbda9d1061913b77a1588695eadb41')], 'directory': hashutil.hex_to_hash( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '6'] ] } } # when self.loader.process_repository( known_state=previous_unfinished_revision) # then # we got the previous run's last revision (rev 6) # so 2 new - self.assertEquals(len(self.loader.all_revisions), 3) + self.assertEquals(len(self.loader.all_revisions), 5) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 1) - last_revision = '99c27ebbd43feca179ac0e895af131d8314cafe1' + last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa - last_revision: '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa + '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa + '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa + last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) occ = self.loader.all_occurrences[0] self.assertEquals(hashutil.hash_to_hex(occ['target']), last_revision) self.assertEquals(occ['origin'], self.origin['id']) class SWHSvnLoaderUpdateWithUnfinishedLoadingChangesButOccurrenceDoneITTest( BaseTestLoader): def setUp(self): super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.loader = SWHSvnLoaderUpdateNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """known repository, swh policy, unfinished revision is less recent than occurrence, we start from last occurrence. """ previous_unfinished_revision = { 'id': hashutil.hex_to_hash( 'a3a577948fdbda9d1061913b77a1588695eadb41'), 'parents': [hashutil.hex_to_hash( '3f51abf3b3d466571be0855dfa67e094f9ceff1b')], 'directory': hashutil.hex_to_hash( '7dc52cc04c3b8bd7c085900d60c159f7b846f866'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '5'] ] } } # when self.loader.process_repository( known_state=previous_unfinished_revision) # then # we got the previous run's last revision (rev 6) # so 2 new - self.assertEquals(len(self.loader.all_revisions), 3) + self.assertEquals(len(self.loader.all_revisions), 5) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 1) - last_revision = '99c27ebbd43feca179ac0e895af131d8314cafe1' + last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa - last_revision: '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa + '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa + '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa + last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) occ = self.loader.all_occurrences[0] self.assertEquals(hashutil.hash_to_hex(occ['target']), last_revision) self.assertEquals(occ['origin'], self.origin['id']) class SWHSvnLoaderUpdateLessRecentNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: Load a known svn repository using the swh policy. The last occurrence seen is less recent than a previous unfinished crawl. """ def swh_previous_revision(self): """Avoid the storage persistence call and return the expected previous revision for that repository. Check the following for explanation about the hashes: - test_loader.org for (swh policy). - cf. SWHSvnLoaderITTest """ return { 'id': hashutil.hex_to_hash( 'a3a577948fdbda9d1061913b77a1588695eadb41'), 'parents': [hashutil.hex_to_hash( '3f51abf3b3d466571be0855dfa67e094f9ceff1b')], 'directory': hashutil.hex_to_hash( '7dc52cc04c3b8bd7c085900d60c159f7b846f866'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '5'] ] } } class SWHSvnLoaderUnfinishedLoadingChangesSinceLastOccurrenceITTest( BaseTestLoader): def setUp(self): super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.loader = SWHSvnLoaderUpdateLessRecentNoStorage( svn_url=self.svn_mirror_url, destination_path=self.destination_path, origin=self.origin) @istest def process_repository(self): """known repository, swh policy, unfinished revision is less recent than occurrence, we start from last occurrence. """ previous_unfinished_revision = { 'id': hashutil.hex_to_hash( '4876cb10aec6f708f7466dddf547567b65f6c39c'), 'parents': [hashutil.hex_to_hash( 'a3a577948fdbda9d1061913b77a1588695eadb41')], 'directory': hashutil.hex_to_hash( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '6'] ] } } # when self.loader.process_repository( known_state=previous_unfinished_revision) # then # we got the previous run's last revision (rev 6) # so 2 new - self.assertEquals(len(self.loader.all_revisions), 3) + self.assertEquals(len(self.loader.all_revisions), 5) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_occurrences), 1) - last_revision = '99c27ebbd43feca179ac0e895af131d8314cafe1' + last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa - last_revision: '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa + '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa + '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa + last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) occ = self.loader.all_occurrences[0] self.assertEquals(hashutil.hash_to_hex(occ['target']), last_revision) self.assertEquals(occ['origin'], self.origin['id'])