Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/svn/tests/test_loader.py
| # Copyright (C) 2016-2018 The Software Heritage developers | # Copyright (C) 2016-2018 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import os | import os | ||||
| from nose.tools import istest | |||||
| from unittest import TestCase | from unittest import TestCase | ||||
| from swh.loader.core.tests import BaseLoaderTest, LoaderNoStorage | |||||
| from swh.loader.svn.loader import (DEFAULT_BRANCH, SvnLoader, | |||||
| SvnLoaderFromRemoteDump, build_swh_snapshot) | |||||
| from swh.model import hashutil | from swh.model import hashutil | ||||
| from swh.loader.svn.loader import build_swh_snapshot, DEFAULT_BRANCH | |||||
| from swh.loader.svn.loader import SvnLoader, SvnLoaderFromRemoteDump | |||||
| from swh.loader.core.tests import LoaderNoStorage, BaseLoaderTest | |||||
| class BaseSvnLoaderTest(BaseLoaderTest): | class BaseSvnLoaderTest(BaseLoaderTest): | ||||
| """Base test loader class. | """Base test loader class. | ||||
| In its setup, it's uncompressing a local svn mirror to /tmp. | In its setup, it's uncompressing a local svn mirror to /tmp. | ||||
| """ | """ | ||||
| def setUp(self, archive_name='pkg-gourmet.tgz', filename='pkg-gourmet'): | def setUp(self, archive_name='pkg-gourmet.tgz', filename='pkg-gourmet'): | ||||
| super().setUp(archive_name=archive_name, filename=filename, | super().setUp(archive_name=archive_name, filename=filename, | ||||
| prefix_tmp_folder_name='swh.loader.svn.', | prefix_tmp_folder_name='swh.loader.svn.', | ||||
| start_path=os.path.dirname(__file__)) | start_path=os.path.dirname(__file__)) | ||||
| self.svn_mirror_url = self.repo_url | self.svn_mirror_url = self.repo_url | ||||
| class TestSnapshot(TestCase): | class TestSnapshot(TestCase): | ||||
| @istest | def test_build_swh_snapshot(self): | ||||
| def build_swh_snapshot(self): | |||||
| actual_snap = build_swh_snapshot('revision-id') | actual_snap = build_swh_snapshot('revision-id') | ||||
| self.assertEquals(actual_snap, { | self.assertEquals(actual_snap, { | ||||
| 'id': None, | 'id': None, | ||||
| 'branches': { | 'branches': { | ||||
| DEFAULT_BRANCH: { | DEFAULT_BRANCH: { | ||||
| 'target': 'revision-id', | 'target': 'revision-id', | ||||
| 'target_type': 'revision', | 'target_type': 'revision', | ||||
| ▲ Show 20 Lines • Show All 115 Lines • ▼ Show 20 Lines | |||||
| class SvnLoaderITest1(BaseSvnLoaderTest): | class SvnLoaderITest1(BaseSvnLoaderTest): | ||||
| """Load an unknown svn repository results in new data. | """Load an unknown svn repository results in new data. | ||||
| """ | """ | ||||
| def setUp(self): | def setUp(self): | ||||
| super().setUp() | super().setUp() | ||||
| self.loader = SvnLoaderNoStorage() | self.loader = SvnLoaderNoStorage() | ||||
| @istest | def test_load(self): | ||||
| def load(self): | |||||
| """Load a new repository results in new swh object and snapshot | """Load a new repository results in new swh object and snapshot | ||||
| """ | """ | ||||
| # when | # when | ||||
| self.loader.load( | self.loader.load( | ||||
| svn_url=self.svn_mirror_url, | svn_url=self.svn_mirror_url, | ||||
| destination_path=self.destination_path) | destination_path=self.destination_path) | ||||
| Show All 26 Lines | class SvnLoaderITest2(BaseSvnLoaderTest): | ||||
| """Load a visited repository with no new change results in no data | """Load a visited repository with no new change results in no data | ||||
| change. | change. | ||||
| """ | """ | ||||
| def setUp(self): | def setUp(self): | ||||
| super().setUp() | super().setUp() | ||||
| self.loader = SvnLoaderUpdateNoStorage() | self.loader = SvnLoaderUpdateNoStorage() | ||||
| @istest | def test_load(self): | ||||
| def load(self): | |||||
| """Load a repository without new changes results in same snapshot | """Load a repository without new changes results in same snapshot | ||||
| """ | """ | ||||
| # when | # when | ||||
| self.loader.load( | self.loader.load( | ||||
| svn_url=self.svn_mirror_url, | svn_url=self.svn_mirror_url, | ||||
| destination_path=self.destination_path) | destination_path=self.destination_path) | ||||
| Show All 18 Lines | """In this scenario, the dump has been tampered with to modify the | ||||
| In effect, that stops the loading and do nothing. | In effect, that stops the loading and do nothing. | ||||
| """ | """ | ||||
| def setUp(self): | def setUp(self): | ||||
| # the svn repository pkg-gourmet has been updated with changes | # the svn repository pkg-gourmet has been updated with changes | ||||
| super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | ||||
| self.loader = SvnLoaderUpdateHistoryAlteredNoStorage() | self.loader = SvnLoaderUpdateHistoryAlteredNoStorage() | ||||
| @istest | def test_load(self): | ||||
| def load(self): | |||||
| """Load known repository with history altered should do nothing | """Load known repository with history altered should do nothing | ||||
| """ | """ | ||||
| # when | # when | ||||
| self.loader.load(svn_url=self.svn_mirror_url, | self.loader.load(svn_url=self.svn_mirror_url, | ||||
| destination_path=self.destination_path) | destination_path=self.destination_path) | ||||
| # then | # then | ||||
| Show All 16 Lines | """In this scenario, the repository has been updated with new changes. | ||||
| snapshot. | snapshot. | ||||
| """ | """ | ||||
| def setUp(self): | def setUp(self): | ||||
| # the svn repository pkg-gourmet has been updated with changes | # the svn repository pkg-gourmet has been updated with changes | ||||
| super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | ||||
| self.loader = SvnLoaderUpdateNoStorage() | self.loader = SvnLoaderUpdateNoStorage() | ||||
| @istest | def test_process_repository(self): | ||||
| def process_repository(self): | |||||
| """Process updated repository should yield new objects | """Process updated repository should yield new objects | ||||
| """ | """ | ||||
| # when | # when | ||||
| self.loader.load(svn_url=self.svn_mirror_url, | self.loader.load(svn_url=self.svn_mirror_url, | ||||
| destination_path=self.destination_path) | destination_path=self.destination_path) | ||||
| # then | # then | ||||
| Show All 30 Lines | """Context: | ||||
| - New visit from scratch done with successful load | - New visit from scratch done with successful load | ||||
| """ | """ | ||||
| def setUp(self): | def setUp(self): | ||||
| # the svn repository pkg-gourmet has been updated with changes | # the svn repository pkg-gourmet has been updated with changes | ||||
| super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | ||||
| self.loader = SvnLoaderUpdateNoStorage() | self.loader = SvnLoaderUpdateNoStorage() | ||||
| @istest | def test_load(self): | ||||
| def load(self): | |||||
| """Load an existing repository from scratch yields same swh objects | """Load an existing repository from scratch yields same swh objects | ||||
| """ | """ | ||||
| # when | # when | ||||
| self.loader.load(svn_url=self.svn_mirror_url, | self.loader.load(svn_url=self.svn_mirror_url, | ||||
| destination_path=self.destination_path, | destination_path=self.destination_path, | ||||
| start_from_scratch=True) | start_from_scratch=True) | ||||
| ▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines | """Context: | ||||
| - Changes on existing repository | - Changes on existing repository | ||||
| - New Visit done with successful new data | - New Visit done with successful new data | ||||
| """ | """ | ||||
| def setUp(self): | def setUp(self): | ||||
| super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | ||||
| self.loader = SvnLoaderWithPreviousRevisionNoStorage() | self.loader = SvnLoaderWithPreviousRevisionNoStorage() | ||||
| @istest | def test_load(self): | ||||
| def load(self): | |||||
| """Load from partial previous visit result in new changes | """Load from partial previous visit result in new changes | ||||
| """ | """ | ||||
| # when | # when | ||||
| self.loader.load( | self.loader.load( | ||||
| svn_url=self.svn_mirror_url, | svn_url=self.svn_mirror_url, | ||||
| destination_path=self.destination_path) | destination_path=self.destination_path) | ||||
| Show All 29 Lines | """Context: | ||||
| - Changes on existing repository | - Changes on existing repository | ||||
| - New Visit done with successful new data | - New Visit done with successful new data | ||||
| """ | """ | ||||
| def setUp(self): | def setUp(self): | ||||
| super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | ||||
| self.loader = SvnLoaderUpdateNoStorage() | self.loader = SvnLoaderUpdateNoStorage() | ||||
| @istest | def test_load(self): | ||||
| def load(self): | |||||
| """Load known and partial repository should start from last visit | """Load known and partial repository should start from last visit | ||||
| """ | """ | ||||
| previous_unfinished_revision = { | previous_unfinished_revision = { | ||||
| 'id': hashutil.hash_to_bytes( | 'id': hashutil.hash_to_bytes( | ||||
| 'a3a577948fdbda9d1061913b77a1588695eadb41'), | 'a3a577948fdbda9d1061913b77a1588695eadb41'), | ||||
| 'parents': [hashutil.hash_to_bytes( | 'parents': [hashutil.hash_to_bytes( | ||||
| '3f51abf3b3d466571be0855dfa67e094f9ceff1b')], | '3f51abf3b3d466571be0855dfa67e094f9ceff1b')], | ||||
| ▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Lines | """Context: | ||||
| - Starting the loading from the last unfinished visit | - Starting the loading from the last unfinished visit | ||||
| - New objects are created (1 new snapshot) | - New objects are created (1 new snapshot) | ||||
| """ | """ | ||||
| def setUp(self): | def setUp(self): | ||||
| super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | ||||
| self.loader = SvnLoaderUpdateLessRecentNoStorage() | self.loader = SvnLoaderUpdateLessRecentNoStorage() | ||||
| @istest | def test_load(self): | ||||
| def load(self): | |||||
| """Load repository should yield revisions starting from last visit | """Load repository should yield revisions starting from last visit | ||||
| """ | """ | ||||
| previous_unfinished_revision = { | previous_unfinished_revision = { | ||||
| 'id': hashutil.hash_to_bytes( | 'id': hashutil.hash_to_bytes( | ||||
| '4876cb10aec6f708f7466dddf547567b65f6c39c'), | '4876cb10aec6f708f7466dddf547567b65f6c39c'), | ||||
| 'parents': [hashutil.hash_to_bytes( | 'parents': [hashutil.hash_to_bytes( | ||||
| 'a3a577948fdbda9d1061913b77a1588695eadb41')], | 'a3a577948fdbda9d1061913b77a1588695eadb41')], | ||||
| ▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | """Check that a svn repo containing a versioned file with CRLF line | ||||
| stored with LF line endings) can be loaded anyway. | stored with LF line endings) can be loaded anyway. | ||||
| """ | """ | ||||
| def setUp(self): | def setUp(self): | ||||
| super().setUp(archive_name='mediawiki-repo-r407-eol-native-crlf.tgz', | super().setUp(archive_name='mediawiki-repo-r407-eol-native-crlf.tgz', | ||||
| filename='mediawiki-repo-r407-eol-native-crlf') | filename='mediawiki-repo-r407-eol-native-crlf') | ||||
| self.loader = SvnLoaderNoStorage() | self.loader = SvnLoaderNoStorage() | ||||
| @istest | def test_process_repository(self): | ||||
| def process_repository(self): | |||||
| """Load repository with CRLF endings (svn:eol-style: native) is ok | """Load repository with CRLF endings (svn:eol-style: native) is ok | ||||
| """ # noqa | """ # noqa | ||||
| # when | # when | ||||
| self.loader.load(svn_url=self.svn_mirror_url, | self.loader.load(svn_url=self.svn_mirror_url, | ||||
| destination_path=self.destination_path) | destination_path=self.destination_path) | ||||
| expected_revisions = { | expected_revisions = { | ||||
| Show All 16 Lines | class SvnLoaderITest10(BaseSvnLoaderTest): # noqa | ||||
| """ | """ | ||||
| def setUp(self): | def setUp(self): | ||||
| super().setUp( | super().setUp( | ||||
| archive_name='pyang-repo-r343-eol-native-mixed-lf-crlf.tgz', | archive_name='pyang-repo-r343-eol-native-mixed-lf-crlf.tgz', | ||||
| filename='pyang-repo-r343-eol-native-mixed-lf-crlf') | filename='pyang-repo-r343-eol-native-mixed-lf-crlf') | ||||
| self.loader = SvnLoaderNoStorage() | self.loader = SvnLoaderNoStorage() | ||||
| @istest | def test_load(self): | ||||
| def load(self): | |||||
| """Load repo with mixed CRLF/LF endings (svn:eol-style:native) is ok | """Load repo with mixed CRLF/LF endings (svn:eol-style:native) is ok | ||||
| """ | """ | ||||
| self.loader.load(svn_url=self.svn_mirror_url, | self.loader.load(svn_url=self.svn_mirror_url, | ||||
| destination_path=self.destination_path) | destination_path=self.destination_path) | ||||
| expected_revisions = { | expected_revisions = { | ||||
| '9c6962eeb9164a636c374be700672355e34a98a7': '16aa6b6271f3456d4643999d234cf39fe3d0cc5a' # noqa | '9c6962eeb9164a636c374be700672355e34a98a7': '16aa6b6271f3456d4643999d234cf39fe3d0cc5a' # noqa | ||||
| Show All 13 Lines | """Context: | ||||
| - Repository with svn:external (which is not deal with for now) | - Repository with svn:external (which is not deal with for now) | ||||
| - Visit is partial with as much data loaded as possible | - Visit is partial with as much data loaded as possible | ||||
| """ | """ | ||||
| def setUp(self): | def setUp(self): | ||||
| super().setUp(archive_name='pkg-gourmet-with-external-id.tgz') | super().setUp(archive_name='pkg-gourmet-with-external-id.tgz') | ||||
| self.loader = SvnLoaderNoStorage() | self.loader = SvnLoaderNoStorage() | ||||
| @istest | def test_load(self): | ||||
| def load(self): | |||||
| """Repository with svn:externals property, will stop raising an error | """Repository with svn:externals property, will stop raising an error | ||||
| """ | """ | ||||
| previous_unfinished_revision = None | previous_unfinished_revision = None | ||||
| # when | # when | ||||
| self.loader.load( | self.loader.load( | ||||
| svn_url=self.svn_mirror_url, | svn_url=self.svn_mirror_url, | ||||
| ▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | """Edge cases: | ||||
| - do the same scenario with symbolic link (instead of file) | - do the same scenario with symbolic link (instead of file) | ||||
| """ | """ | ||||
| def setUp(self): | def setUp(self): | ||||
| super().setUp( | super().setUp( | ||||
| archive_name='pkg-gourmet-with-edge-case-links-and-files.tgz') | archive_name='pkg-gourmet-with-edge-case-links-and-files.tgz') | ||||
| self.loader = SvnLoaderNoStorage() | self.loader = SvnLoaderNoStorage() | ||||
| @istest | def test_load(self): | ||||
| def load(self): | |||||
| """File/Link removed prior to folder with same name creation is ok | """File/Link removed prior to folder with same name creation is ok | ||||
| """ | """ | ||||
| previous_unfinished_revision = None | previous_unfinished_revision = None | ||||
| # when | # when | ||||
| self.loader.load( | self.loader.load( | ||||
| svn_url=self.svn_mirror_url, | svn_url=self.svn_mirror_url, | ||||
| ▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | """Edge cases: | ||||
| - wrong symbolic link with empty space names | - wrong symbolic link with empty space names | ||||
| """ | """ | ||||
| def setUp(self): | def setUp(self): | ||||
| super().setUp( | super().setUp( | ||||
| archive_name='pkg-gourmet-with-wrong-link-cases.tgz') | archive_name='pkg-gourmet-with-wrong-link-cases.tgz') | ||||
| self.loader = SvnLoaderNoStorage() | self.loader = SvnLoaderNoStorage() | ||||
| @istest | def test_load(self): | ||||
| def load(self): | |||||
| """Wrong link or empty space-named link should be ok | """Wrong link or empty space-named link should be ok | ||||
| """ | """ | ||||
| # when | # when | ||||
| self.loader.load( | self.loader.load( | ||||
| svn_url=self.svn_mirror_url, | svn_url=self.svn_mirror_url, | ||||
| destination_path=self.destination_path) | destination_path=self.destination_path) | ||||
| ▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines | |||||
| class SvnLoaderFromRemoteDump(BaseSvnLoaderTest): | class SvnLoaderFromRemoteDump(BaseSvnLoaderTest): | ||||
| """ | """ | ||||
| Check that the results obtained with the remote svn dump loader | Check that the results obtained with the remote svn dump loader | ||||
| and the base svn loader are the same. | and the base svn loader are the same. | ||||
| """ | """ | ||||
| def setUp(self): | def setUp(self): | ||||
| super().setUp(archive_name='pkg-gourmet.tgz') | super().setUp(archive_name='pkg-gourmet.tgz') | ||||
| @istest | def test_load(self): | ||||
| def load(self): | |||||
| """ | """ | ||||
| Compare results of remote dump loader and base loader | Compare results of remote dump loader and base loader | ||||
| """ | """ | ||||
| dump_loader = SvnLoaderFromRemoteDumpNoStorage() | dump_loader = SvnLoaderFromRemoteDumpNoStorage() | ||||
| dump_loader.load(svn_url=self.svn_mirror_url) | dump_loader.load(svn_url=self.svn_mirror_url) | ||||
| base_loader = SvnLoaderNoStorage() | base_loader = SvnLoaderNoStorage() | ||||
| base_loader.load(svn_url=self.svn_mirror_url) | base_loader.load(svn_url=self.svn_mirror_url) | ||||
| Show All 13 Lines | """Edge cases: The repository held some user defined svn-properties | ||||
| with special encodings, this prevented the repository from | with special encodings, this prevented the repository from | ||||
| being loaded even though we do not ingest those information. | being loaded even though we do not ingest those information. | ||||
| """ | """ | ||||
| def setUp(self): | def setUp(self): | ||||
| super().setUp(archive_name='httthttt.tgz', filename='httthttt') | super().setUp(archive_name='httthttt.tgz', filename='httthttt') | ||||
| self.loader = SvnLoaderNoStorage() | self.loader = SvnLoaderNoStorage() | ||||
| @istest | def test_load(self): | ||||
| def load(self): | |||||
| """Decoding user defined svn properties error should not fail loading | """Decoding user defined svn properties error should not fail loading | ||||
| """ | """ | ||||
| # when | # when | ||||
| self.loader.load( | self.loader.load( | ||||
| svn_url=self.svn_mirror_url, | svn_url=self.svn_mirror_url, | ||||
| destination_path=self.destination_path) | destination_path=self.destination_path) | ||||
| Show All 29 Lines | |||||