diff --git a/debian/control b/debian/control --- a/debian/control +++ b/debian/control @@ -12,10 +12,10 @@ python3-setuptools, python3-subvertpy (>= 0.9.4~), python3-swh.core (>= 0.0.19~), - python3-swh.loader.core (>= 0.0.35~), + python3-swh.loader.core (>= 0.0.36~), python3-swh.model (>= 0.0.18~), python3-swh.scheduler (>= 0.0.14~), - python3-swh.storage (>= 0.0.83~), + python3-swh.storage (>= 0.0.114~), python3-vcversioner, subversion Standards-Version: 3.9.6 @@ -25,10 +25,10 @@ Architecture: all Depends: gzip, python3-swh.core (>= 0.0.19~), - python3-swh.loader.core (>= 0.0.35~), + python3-swh.loader.core (>= 0.0.36~), python3-swh.model (>= 0.0.18~), python3-swh.scheduler (>= 0.0.14~), - python3-swh.storage (>= 0.0.83~), + python3-swh.storage (>= 0.0.114~), subversion, ${misc:Depends}, ${python3:Depends} diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,5 +1,5 @@ swh.core >= 0.0.19 -swh.storage >= 0.0.83 +swh.storage >= 0.0.114 swh.model >= 0.0.18 swh.scheduler >= 0.0.14 -swh.loader.core >= 0.0.35 +swh.loader.core >= 0.0.36 diff --git a/swh/loader/svn/tests/test_loader.py b/swh/loader/svn/tests/test_loader.py --- a/swh/loader/svn/tests/test_loader.py +++ b/swh/loader/svn/tests/test_loader.py @@ -6,7 +6,7 @@ import os from unittest import TestCase -from swh.loader.core.tests import BaseLoaderTest, LoaderNoStorage +from swh.loader.core.tests import BaseLoaderTest from swh.loader.svn.loader import (DEFAULT_BRANCH, SvnLoader, SvnLoaderFromRemoteDump, build_swh_snapshot) from swh.model import hashutil @@ -18,11 +18,16 @@ In its setup, it's uncompressing a local svn mirror to /tmp. """ - def setUp(self, archive_name='pkg-gourmet.tgz', filename='pkg-gourmet'): + def setUp(self, archive_name='pkg-gourmet.tgz', filename='pkg-gourmet', + loader=None): super().setUp(archive_name=archive_name, filename=filename, prefix_tmp_folder_name='swh.loader.svn.', start_path=os.path.dirname(__file__)) self.svn_mirror_url = self.repo_url + self.loader = loader + if not loader: + self.loader = SvnLoaderMemoryStorage() + self.storage = self.loader.storage class TestSnapshot(TestCase): @@ -40,6 +45,31 @@ }) +_LOADER_TEST_CONFIG = { + 'check_revision': {'limit': 100, 'status': False}, + 'content_packet_block_size_bytes': 104857600, + 'content_packet_size': 10000, + 'content_packet_size_bytes': 1073741824, + 'content_size_limit': 104857600, + 'debug': False, + 'directory_packet_size': 2500, + 'log_db': 'dbname=softwareheritage-log', + 'occurrence_packet_size': 1000, + 'release_packet_size': 1000, + 'revision_packet_size': 10, + 'save_data': False, + 'save_data_path': '', + 'send_contents': True, + 'send_directories': True, + 'send_occurrences': True, + 'send_releases': True, + 'send_revisions': True, + 'send_snapshot': True, + 'storage': {'args': {}, 'cls': 'memory'}, + 'temp_directory': '/home/antoine/tmp/' +} + + class LoaderWithState: """Additional state setup (bypassed by some override for test purposes) @@ -56,8 +86,11 @@ 'visit': 1, } + def parse_config_file(self, *args, **kwargs): + return _LOADER_TEST_CONFIG + -class SvnLoaderNoStorage(LoaderNoStorage, LoaderWithState, SvnLoader): +class SvnLoaderMemoryStorage(LoaderWithState, SvnLoader): """An SVNLoader with no persistence. Context: @@ -71,7 +104,7 @@ return {} -class SvnLoaderUpdateNoStorage(LoaderNoStorage, LoaderWithState, SvnLoader): +class SvnLoaderUpdateMemoryStorage(LoaderWithState, SvnLoader): """An SVNLoader with no persistence. Context: @@ -93,7 +126,10 @@ """ return { - 'snapshot': 'something', # need a snapshot of sort + 'snapshot': { + 'id': 'something', + 'branches': {} + }, # need a snapshot of sort 'revision': { 'id': hashutil.hash_to_bytes( '4876cb10aec6f708f7466dddf547567b65f6c39c'), @@ -113,8 +149,8 @@ } -class SvnLoaderUpdateHistoryAlteredNoStorage(LoaderNoStorage, LoaderWithState, - SvnLoader): +class SvnLoaderUpdateHistoryAlteredMemoryStorage(LoaderWithState, + SvnLoader): """Context: Load a known svn repository using the swh policy with its history altered so we do not update it. @@ -156,7 +192,6 @@ """ def setUp(self): super().setUp() - self.loader = SvnLoaderNoStorage() def test_load(self): """Load a new repository results in new swh object and snapshot @@ -184,7 +219,7 @@ last_revision: '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa } - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) self.assertCountSnapshots(1) # FIXME: Check the snapshot's state # self.assertEqual(self.loader.all_snapshots[0], {}) @@ -198,8 +233,7 @@ """ def setUp(self): - super().setUp() - self.loader = SvnLoaderUpdateNoStorage() + super().setUp(loader=SvnLoaderUpdateMemoryStorage()) def test_load(self): """Load a repository without new changes results in same snapshot @@ -233,8 +267,8 @@ """ def setUp(self): # the svn repository pkg-gourmet has been updated with changes - super().setUp(archive_name='pkg-gourmet-with-updates.tgz') - self.loader = SvnLoaderUpdateHistoryAlteredNoStorage() + super().setUp(archive_name='pkg-gourmet-with-updates.tgz', + loader=SvnLoaderUpdateHistoryAlteredMemoryStorage()) def test_load(self): """Load known repository with history altered should do nothing @@ -266,8 +300,8 @@ """ def setUp(self): # the svn repository pkg-gourmet has been updated with changes - super().setUp(archive_name='pkg-gourmet-with-updates.tgz') - self.loader = SvnLoaderUpdateNoStorage() + super().setUp(archive_name='pkg-gourmet-with-updates.tgz', + loader=SvnLoaderUpdateMemoryStorage()) def test_process_repository(self): """Process updated repository should yield new objects @@ -295,7 +329,7 @@ last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) self.assertCountSnapshots(1) # FIXME: Check the snapshot's state @@ -313,8 +347,8 @@ """ def setUp(self): # the svn repository pkg-gourmet has been updated with changes - super().setUp(archive_name='pkg-gourmet-with-updates.tgz') - self.loader = SvnLoaderUpdateNoStorage() + super().setUp(archive_name='pkg-gourmet-with-updates.tgz', + loader=SvnLoaderUpdateMemoryStorage()) def test_load(self): """Load an existing repository from scratch yields same swh objects @@ -346,7 +380,7 @@ '171dc35522bfd17dda4e90a542a0377fb2fc707a': 'fd24a76c87a3207428e06612b49860fc78e9f6dc', # noqa } - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) self.assertCountSnapshots(1) # FIXME: Check the snapshot's state @@ -355,8 +389,8 @@ self.assertEqual(self.loader.visit_status(), 'full') -class SvnLoaderWithPreviousRevisionNoStorage(LoaderNoStorage, LoaderWithState, - SvnLoader): +class SvnLoaderWithPreviousRevisionMemoryStorage(LoaderWithState, + SvnLoader): """An SVNLoader with no persistence. Context: Load a known svn repository using the swh policy with its @@ -400,8 +434,8 @@ """ def setUp(self): - super().setUp(archive_name='pkg-gourmet-with-updates.tgz') - self.loader = SvnLoaderWithPreviousRevisionNoStorage() + super().setUp(archive_name='pkg-gourmet-with-updates.tgz', + loader=SvnLoaderWithPreviousRevisionMemoryStorage()) def test_load(self): """Load from partial previous visit result in new changes @@ -430,7 +464,7 @@ last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) self.assertCountSnapshots(1) # FIXME: Check the snapshot's state # self.assertEqual(self.loader.all_snapshots[0], {}) @@ -446,8 +480,8 @@ """ def setUp(self): - super().setUp(archive_name='pkg-gourmet-with-updates.tgz') - self.loader = SvnLoaderUpdateNoStorage() + super().setUp(archive_name='pkg-gourmet-with-updates.tgz', + loader=SvnLoaderUpdateMemoryStorage()) def test_load(self): """Load known and partial repository should start from last visit @@ -493,7 +527,7 @@ last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) self.assertCountSnapshots(1) # FIXME: Check the snapshot's state # self.assertEqual(self.loader.all_snapshots[0], {}) @@ -501,8 +535,8 @@ self.assertEqual(self.loader.visit_status(), 'full') -class SvnLoaderUpdateLessRecentNoStorage(LoaderNoStorage, LoaderWithState, - SvnLoader): +class SvnLoaderUpdateLessRecentMemoryStorage(LoaderWithState, + SvnLoader): """Context: Load a known svn repository. The last visit seen is less recent than a previous unfinished crawl. @@ -547,8 +581,8 @@ """ def setUp(self): - super().setUp(archive_name='pkg-gourmet-with-updates.tgz') - self.loader = SvnLoaderUpdateLessRecentNoStorage() + super().setUp(archive_name='pkg-gourmet-with-updates.tgz', + loader=SvnLoaderUpdateLessRecentMemoryStorage()) def test_load(self): """Load repository should yield revisions starting from last visit @@ -593,7 +627,7 @@ last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) self.assertCountSnapshots(1) # FIXME: Check the snapshot's state # self.assertEqual(self.loader.all_snapshots[0], {}) @@ -610,8 +644,8 @@ """ def setUp(self): super().setUp(archive_name='mediawiki-repo-r407-eol-native-crlf.tgz', - filename='mediawiki-repo-r407-eol-native-crlf') - self.loader = SvnLoaderNoStorage() + filename='mediawiki-repo-r407-eol-native-crlf', + loader=SvnLoaderMemoryStorage()) def test_process_repository(self): """Load repository with CRLF endings (svn:eol-style: native) is ok @@ -624,7 +658,7 @@ expected_revisions = { '7da4975c363101b819756d33459f30a866d01b1b': 'f63637223ee0f7d4951ffd2d4d9547a4882c5d8b' # noqa } - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) self.assertCountSnapshots(1) # FIXME: Check the snapshot's state # self.assertEqual(self.loader.all_snapshots[0], {}) @@ -643,8 +677,8 @@ def setUp(self): super().setUp( archive_name='pyang-repo-r343-eol-native-mixed-lf-crlf.tgz', - filename='pyang-repo-r343-eol-native-mixed-lf-crlf') - self.loader = SvnLoaderNoStorage() + filename='pyang-repo-r343-eol-native-mixed-lf-crlf', + loader=SvnLoaderMemoryStorage()) def test_load(self): """Load repo with mixed CRLF/LF endings (svn:eol-style:native) is ok @@ -657,7 +691,7 @@ '9c6962eeb9164a636c374be700672355e34a98a7': '16aa6b6271f3456d4643999d234cf39fe3d0cc5a' # noqa } - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) self.assertCountSnapshots(1) # FIXME: Check the snapshot's state # self.assertEqual(self.loader.all_snapshots[0], {}) @@ -673,8 +707,8 @@ """ def setUp(self): - super().setUp(archive_name='pkg-gourmet-with-external-id.tgz') - self.loader = SvnLoaderNoStorage() + super().setUp(archive_name='pkg-gourmet-with-external-id.tgz', + loader=SvnLoaderMemoryStorage()) def test_load(self): """Repository with svn:externals property, will stop raising an error @@ -720,7 +754,7 @@ } # The last revision being the one used later to start back from - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) self.assertCountSnapshots(1) # FIXME: Check the snapshot's state self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) @@ -736,8 +770,8 @@ """ def setUp(self): super().setUp( - archive_name='pkg-gourmet-with-edge-case-links-and-files.tgz') - self.loader = SvnLoaderNoStorage() + archive_name='pkg-gourmet-with-edge-case-links-and-files.tgz', + loader=SvnLoaderMemoryStorage()) def test_load(self): """File/Link removed prior to folder with same name creation is ok @@ -779,7 +813,7 @@ last_revision: '6b1e0243768ff9ac060064b2eeade77e764ffc82', # noqa } - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) self.assertCountSnapshots(1) # FIXME: Check the snapshot's state self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) @@ -794,8 +828,8 @@ """ def setUp(self): super().setUp( - archive_name='pkg-gourmet-with-wrong-link-cases.tgz') - self.loader = SvnLoaderNoStorage() + archive_name='pkg-gourmet-with-wrong-link-cases.tgz', + loader=SvnLoaderMemoryStorage()) def test_load(self): """Wrong link or empty space-named link should be ok @@ -837,15 +871,15 @@ last_revision: 'd853d9628f6f0008d324fed27dadad00ce48bc62', # noqa } - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) self.assertCountSnapshots(1) # FIXME: Check the snapshot's state self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.visit_status(), 'full') -class SvnLoaderFromRemoteDumpNoStorage(LoaderNoStorage, LoaderWithState, - SvnLoaderFromRemoteDump): +class SvnLoaderFromRemoteDumpMemoryStorage(LoaderWithState, + SvnLoaderFromRemoteDump): """A SvnLoaderFromRemoteDump with no persistence. Context: @@ -866,26 +900,28 @@ and the base svn loader are the same. """ def setUp(self): - super().setUp(archive_name='pkg-gourmet.tgz') + super().setUp(archive_name='pkg-gourmet.tgz', + loader=SvnLoaderFromRemoteDumpMemoryStorage()) def test_load(self): """ Compare results of remote dump loader and base loader """ - dump_loader = SvnLoaderFromRemoteDumpNoStorage() + dump_loader = self.loader dump_loader.load(svn_url=self.svn_mirror_url) - base_loader = SvnLoaderNoStorage() + self.assertCountContents(19) + self.assertCountDirectories(17) + self.assertCountRevisions(6) + self.assertCountSnapshots(1) + + base_loader = SvnLoaderMemoryStorage() base_loader.load(svn_url=self.svn_mirror_url) - self.assertEqual(dump_loader.state('content'), - base_loader.state('content')) - self.assertEqual(dump_loader.state('directory'), - base_loader.state('directory')) - self.assertEqual(dump_loader.state('revision'), - base_loader.state('revision')) - self.assertEqual(dump_loader.state('snapshot'), - base_loader.state('snapshot')) + dump_storage_stat = dump_loader.storage.stat_counters() + base_storage_stat = base_loader.storage.stat_counters() + + self.assertEqual(dump_storage_stat, base_storage_stat) class SvnLoaderITTest14(BaseSvnLoaderTest): @@ -895,8 +931,8 @@ """ def setUp(self): - super().setUp(archive_name='httthttt.tgz', filename='httthttt') - self.loader = SvnLoaderNoStorage() + super().setUp(archive_name='httthttt.tgz', filename='httthttt', + loader=SvnLoaderMemoryStorage()) def test_load(self): """Decoding user defined svn properties error should not fail loading @@ -922,7 +958,7 @@ last_revision: 'f051d60256b2d89a0ca2704d6f91ad1b0ab44e02', } - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) expected_snapshot_id = '70487267f682c07e52a2371061369b6cf5bffa47' expected_branches = { @@ -932,7 +968,7 @@ } } - self.assertSnapshotOk(expected_snapshot_id, expected_branches) + self.assertSnapshotEqual(expected_snapshot_id, expected_branches) self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.visit_status(), 'full')