diff --git a/debian/control b/debian/control
--- a/debian/control
+++ b/debian/control
@@ -12,10 +12,10 @@
python3-setuptools,
python3-subvertpy (>= 0.9.4~),
python3-swh.core (>= 0.0.19~),
- python3-swh.loader.core (>= 0.0.35~),
+ python3-swh.loader.core (>= 0.0.36~),
python3-swh.model (>= 0.0.18~),
python3-swh.scheduler (>= 0.0.14~),
- python3-swh.storage (>= 0.0.83~),
+ python3-swh.storage (>= 0.0.114~),
python3-vcversioner,
subversion
Standards-Version: 3.9.6
@@ -25,10 +25,10 @@
Architecture: all
Depends: gzip,
python3-swh.core (>= 0.0.19~),
- python3-swh.loader.core (>= 0.0.35~),
+ python3-swh.loader.core (>= 0.0.36~),
python3-swh.model (>= 0.0.18~),
python3-swh.scheduler (>= 0.0.14~),
- python3-swh.storage (>= 0.0.83~),
+ python3-swh.storage (>= 0.0.114~),
subversion,
${misc:Depends},
${python3:Depends}
diff --git a/requirements-swh.txt b/requirements-swh.txt
--- a/requirements-swh.txt
+++ b/requirements-swh.txt
@@ -1,5 +1,5 @@
swh.core >= 0.0.19
-swh.storage >= 0.0.83
+swh.storage >= 0.0.114
swh.model >= 0.0.18
swh.scheduler >= 0.0.14
-swh.loader.core >= 0.0.35
+swh.loader.core >= 0.0.36
diff --git a/swh/loader/svn/tests/test_loader.py b/swh/loader/svn/tests/test_loader.py
--- a/swh/loader/svn/tests/test_loader.py
+++ b/swh/loader/svn/tests/test_loader.py
@@ -6,7 +6,7 @@
import os
from unittest import TestCase
-from swh.loader.core.tests import BaseLoaderTest, LoaderNoStorage
+from swh.loader.core.tests import BaseLoaderTest
from swh.loader.svn.loader import (DEFAULT_BRANCH, SvnLoader,
SvnLoaderFromRemoteDump, build_swh_snapshot)
from swh.model import hashutil
@@ -18,11 +18,16 @@
In its setup, it's uncompressing a local svn mirror to /tmp.
"""
- def setUp(self, archive_name='pkg-gourmet.tgz', filename='pkg-gourmet'):
+ def setUp(self, archive_name='pkg-gourmet.tgz', filename='pkg-gourmet',
+ loader=None):
super().setUp(archive_name=archive_name, filename=filename,
prefix_tmp_folder_name='swh.loader.svn.',
start_path=os.path.dirname(__file__))
self.svn_mirror_url = self.repo_url
+ self.loader = loader
+ if not loader:
+ self.loader = SvnLoaderMemoryStorage()
+ self.storage = self.loader.storage
class TestSnapshot(TestCase):
@@ -40,6 +45,31 @@
})
+_LOADER_TEST_CONFIG = {
+ 'check_revision': {'limit': 100, 'status': False},
+ 'content_packet_block_size_bytes': 104857600,
+ 'content_packet_size': 10000,
+ 'content_packet_size_bytes': 1073741824,
+ 'content_size_limit': 104857600,
+ 'debug': False,
+ 'directory_packet_size': 2500,
+ 'log_db': 'dbname=softwareheritage-log',
+ 'occurrence_packet_size': 1000,
+ 'release_packet_size': 1000,
+ 'revision_packet_size': 10,
+ 'save_data': False,
+ 'save_data_path': '',
+ 'send_contents': True,
+ 'send_directories': True,
+ 'send_occurrences': True,
+ 'send_releases': True,
+ 'send_revisions': True,
+ 'send_snapshot': True,
+ 'storage': {'args': {}, 'cls': 'memory'},
+ 'temp_directory': '/home/antoine/tmp/'
+}
+
+
class LoaderWithState:
"""Additional state setup (bypassed by some override for test purposes)
@@ -56,8 +86,11 @@
'visit': 1,
}
+ def parse_config_file(self, *args, **kwargs):
+ return _LOADER_TEST_CONFIG
+
-class SvnLoaderNoStorage(LoaderNoStorage, LoaderWithState, SvnLoader):
+class SvnLoaderMemoryStorage(LoaderWithState, SvnLoader):
"""An SVNLoader with no persistence.
Context:
@@ -71,7 +104,7 @@
return {}
-class SvnLoaderUpdateNoStorage(LoaderNoStorage, LoaderWithState, SvnLoader):
+class SvnLoaderUpdateMemoryStorage(LoaderWithState, SvnLoader):
"""An SVNLoader with no persistence.
Context:
@@ -93,7 +126,10 @@
"""
return {
- 'snapshot': 'something', # need a snapshot of sort
+ 'snapshot': {
+ 'id': 'something',
+ 'branches': {}
+ }, # need a snapshot of sort
'revision': {
'id': hashutil.hash_to_bytes(
'4876cb10aec6f708f7466dddf547567b65f6c39c'),
@@ -113,8 +149,8 @@
}
-class SvnLoaderUpdateHistoryAlteredNoStorage(LoaderNoStorage, LoaderWithState,
- SvnLoader):
+class SvnLoaderUpdateHistoryAlteredMemoryStorage(LoaderWithState,
+ SvnLoader):
"""Context: Load a known svn repository using the swh policy with its
history altered so we do not update it.
@@ -156,7 +192,6 @@
"""
def setUp(self):
super().setUp()
- self.loader = SvnLoaderNoStorage()
def test_load(self):
"""Load a new repository results in new swh object and snapshot
@@ -184,7 +219,7 @@
last_revision: '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa
}
- self.assertRevisionsOk(expected_revisions)
+ self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state
# self.assertEqual(self.loader.all_snapshots[0], {})
@@ -198,8 +233,7 @@
"""
def setUp(self):
- super().setUp()
- self.loader = SvnLoaderUpdateNoStorage()
+ super().setUp(loader=SvnLoaderUpdateMemoryStorage())
def test_load(self):
"""Load a repository without new changes results in same snapshot
@@ -233,8 +267,8 @@
"""
def setUp(self):
# the svn repository pkg-gourmet has been updated with changes
- super().setUp(archive_name='pkg-gourmet-with-updates.tgz')
- self.loader = SvnLoaderUpdateHistoryAlteredNoStorage()
+ super().setUp(archive_name='pkg-gourmet-with-updates.tgz',
+ loader=SvnLoaderUpdateHistoryAlteredMemoryStorage())
def test_load(self):
"""Load known repository with history altered should do nothing
@@ -266,8 +300,8 @@
"""
def setUp(self):
# the svn repository pkg-gourmet has been updated with changes
- super().setUp(archive_name='pkg-gourmet-with-updates.tgz')
- self.loader = SvnLoaderUpdateNoStorage()
+ super().setUp(archive_name='pkg-gourmet-with-updates.tgz',
+ loader=SvnLoaderUpdateMemoryStorage())
def test_process_repository(self):
"""Process updated repository should yield new objects
@@ -295,7 +329,7 @@
last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa
}
- self.assertRevisionsOk(expected_revisions)
+ self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state
@@ -313,8 +347,8 @@
"""
def setUp(self):
# the svn repository pkg-gourmet has been updated with changes
- super().setUp(archive_name='pkg-gourmet-with-updates.tgz')
- self.loader = SvnLoaderUpdateNoStorage()
+ super().setUp(archive_name='pkg-gourmet-with-updates.tgz',
+ loader=SvnLoaderUpdateMemoryStorage())
def test_load(self):
"""Load an existing repository from scratch yields same swh objects
@@ -346,7 +380,7 @@
'171dc35522bfd17dda4e90a542a0377fb2fc707a': 'fd24a76c87a3207428e06612b49860fc78e9f6dc', # noqa
}
- self.assertRevisionsOk(expected_revisions)
+ self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state
@@ -355,8 +389,8 @@
self.assertEqual(self.loader.visit_status(), 'full')
-class SvnLoaderWithPreviousRevisionNoStorage(LoaderNoStorage, LoaderWithState,
- SvnLoader):
+class SvnLoaderWithPreviousRevisionMemoryStorage(LoaderWithState,
+ SvnLoader):
"""An SVNLoader with no persistence.
Context: Load a known svn repository using the swh policy with its
@@ -400,8 +434,8 @@
"""
def setUp(self):
- super().setUp(archive_name='pkg-gourmet-with-updates.tgz')
- self.loader = SvnLoaderWithPreviousRevisionNoStorage()
+ super().setUp(archive_name='pkg-gourmet-with-updates.tgz',
+ loader=SvnLoaderWithPreviousRevisionMemoryStorage())
def test_load(self):
"""Load from partial previous visit result in new changes
@@ -430,7 +464,7 @@
last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa
}
- self.assertRevisionsOk(expected_revisions)
+ self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state
# self.assertEqual(self.loader.all_snapshots[0], {})
@@ -446,8 +480,8 @@
"""
def setUp(self):
- super().setUp(archive_name='pkg-gourmet-with-updates.tgz')
- self.loader = SvnLoaderUpdateNoStorage()
+ super().setUp(archive_name='pkg-gourmet-with-updates.tgz',
+ loader=SvnLoaderUpdateMemoryStorage())
def test_load(self):
"""Load known and partial repository should start from last visit
@@ -493,7 +527,7 @@
last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa
}
- self.assertRevisionsOk(expected_revisions)
+ self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state
# self.assertEqual(self.loader.all_snapshots[0], {})
@@ -501,8 +535,8 @@
self.assertEqual(self.loader.visit_status(), 'full')
-class SvnLoaderUpdateLessRecentNoStorage(LoaderNoStorage, LoaderWithState,
- SvnLoader):
+class SvnLoaderUpdateLessRecentMemoryStorage(LoaderWithState,
+ SvnLoader):
"""Context:
Load a known svn repository. The last visit seen is less
recent than a previous unfinished crawl.
@@ -547,8 +581,8 @@
"""
def setUp(self):
- super().setUp(archive_name='pkg-gourmet-with-updates.tgz')
- self.loader = SvnLoaderUpdateLessRecentNoStorage()
+ super().setUp(archive_name='pkg-gourmet-with-updates.tgz',
+ loader=SvnLoaderUpdateLessRecentMemoryStorage())
def test_load(self):
"""Load repository should yield revisions starting from last visit
@@ -593,7 +627,7 @@
last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa
}
- self.assertRevisionsOk(expected_revisions)
+ self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state
# self.assertEqual(self.loader.all_snapshots[0], {})
@@ -610,8 +644,8 @@
"""
def setUp(self):
super().setUp(archive_name='mediawiki-repo-r407-eol-native-crlf.tgz',
- filename='mediawiki-repo-r407-eol-native-crlf')
- self.loader = SvnLoaderNoStorage()
+ filename='mediawiki-repo-r407-eol-native-crlf',
+ loader=SvnLoaderMemoryStorage())
def test_process_repository(self):
"""Load repository with CRLF endings (svn:eol-style: native) is ok
@@ -624,7 +658,7 @@
expected_revisions = {
'7da4975c363101b819756d33459f30a866d01b1b': 'f63637223ee0f7d4951ffd2d4d9547a4882c5d8b' # noqa
}
- self.assertRevisionsOk(expected_revisions)
+ self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state
# self.assertEqual(self.loader.all_snapshots[0], {})
@@ -643,8 +677,8 @@
def setUp(self):
super().setUp(
archive_name='pyang-repo-r343-eol-native-mixed-lf-crlf.tgz',
- filename='pyang-repo-r343-eol-native-mixed-lf-crlf')
- self.loader = SvnLoaderNoStorage()
+ filename='pyang-repo-r343-eol-native-mixed-lf-crlf',
+ loader=SvnLoaderMemoryStorage())
def test_load(self):
"""Load repo with mixed CRLF/LF endings (svn:eol-style:native) is ok
@@ -657,7 +691,7 @@
'9c6962eeb9164a636c374be700672355e34a98a7': '16aa6b6271f3456d4643999d234cf39fe3d0cc5a' # noqa
}
- self.assertRevisionsOk(expected_revisions)
+ self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state
# self.assertEqual(self.loader.all_snapshots[0], {})
@@ -673,8 +707,8 @@
"""
def setUp(self):
- super().setUp(archive_name='pkg-gourmet-with-external-id.tgz')
- self.loader = SvnLoaderNoStorage()
+ super().setUp(archive_name='pkg-gourmet-with-external-id.tgz',
+ loader=SvnLoaderMemoryStorage())
def test_load(self):
"""Repository with svn:externals property, will stop raising an error
@@ -720,7 +754,7 @@
}
# The last revision being the one used later to start back from
- self.assertRevisionsOk(expected_revisions)
+ self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state
self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
@@ -736,8 +770,8 @@
"""
def setUp(self):
super().setUp(
- archive_name='pkg-gourmet-with-edge-case-links-and-files.tgz')
- self.loader = SvnLoaderNoStorage()
+ archive_name='pkg-gourmet-with-edge-case-links-and-files.tgz',
+ loader=SvnLoaderMemoryStorage())
def test_load(self):
"""File/Link removed prior to folder with same name creation is ok
@@ -779,7 +813,7 @@
last_revision: '6b1e0243768ff9ac060064b2eeade77e764ffc82', # noqa
}
- self.assertRevisionsOk(expected_revisions)
+ self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state
self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
@@ -794,8 +828,8 @@
"""
def setUp(self):
super().setUp(
- archive_name='pkg-gourmet-with-wrong-link-cases.tgz')
- self.loader = SvnLoaderNoStorage()
+ archive_name='pkg-gourmet-with-wrong-link-cases.tgz',
+ loader=SvnLoaderMemoryStorage())
def test_load(self):
"""Wrong link or empty space-named link should be ok
@@ -837,15 +871,15 @@
last_revision: 'd853d9628f6f0008d324fed27dadad00ce48bc62', # noqa
}
- self.assertRevisionsOk(expected_revisions)
+ self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state
self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full')
-class SvnLoaderFromRemoteDumpNoStorage(LoaderNoStorage, LoaderWithState,
- SvnLoaderFromRemoteDump):
+class SvnLoaderFromRemoteDumpMemoryStorage(LoaderWithState,
+ SvnLoaderFromRemoteDump):
"""A SvnLoaderFromRemoteDump with no persistence.
Context:
@@ -866,26 +900,28 @@
and the base svn loader are the same.
"""
def setUp(self):
- super().setUp(archive_name='pkg-gourmet.tgz')
+ super().setUp(archive_name='pkg-gourmet.tgz',
+ loader=SvnLoaderFromRemoteDumpMemoryStorage())
def test_load(self):
"""
Compare results of remote dump loader and base loader
"""
- dump_loader = SvnLoaderFromRemoteDumpNoStorage()
+ dump_loader = self.loader
dump_loader.load(svn_url=self.svn_mirror_url)
- base_loader = SvnLoaderNoStorage()
+ self.assertCountContents(19)
+ self.assertCountDirectories(17)
+ self.assertCountRevisions(6)
+ self.assertCountSnapshots(1)
+
+ base_loader = SvnLoaderMemoryStorage()
base_loader.load(svn_url=self.svn_mirror_url)
- self.assertEqual(dump_loader.state('content'),
- base_loader.state('content'))
- self.assertEqual(dump_loader.state('directory'),
- base_loader.state('directory'))
- self.assertEqual(dump_loader.state('revision'),
- base_loader.state('revision'))
- self.assertEqual(dump_loader.state('snapshot'),
- base_loader.state('snapshot'))
+ dump_storage_stat = dump_loader.storage.stat_counters()
+ base_storage_stat = base_loader.storage.stat_counters()
+
+ self.assertEqual(dump_storage_stat, base_storage_stat)
class SvnLoaderITTest14(BaseSvnLoaderTest):
@@ -895,8 +931,8 @@
"""
def setUp(self):
- super().setUp(archive_name='httthttt.tgz', filename='httthttt')
- self.loader = SvnLoaderNoStorage()
+ super().setUp(archive_name='httthttt.tgz', filename='httthttt',
+ loader=SvnLoaderMemoryStorage())
def test_load(self):
"""Decoding user defined svn properties error should not fail loading
@@ -922,7 +958,7 @@
last_revision: 'f051d60256b2d89a0ca2704d6f91ad1b0ab44e02',
}
- self.assertRevisionsOk(expected_revisions)
+ self.assertRevisionsContain(expected_revisions)
expected_snapshot_id = '70487267f682c07e52a2371061369b6cf5bffa47'
expected_branches = {
@@ -932,7 +968,7 @@
}
}
- self.assertSnapshotOk(expected_snapshot_id, expected_branches)
+ self.assertSnapshotEqual(expected_snapshot_id, expected_branches)
self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full')