Changeset View
Standalone View
swh/loader/svn/tests/test_loader.py
# Copyright (C) 2016-2018 The Software Heritage developers | # Copyright (C) 2016-2018 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import copy | |||||
import os | import os | ||||
from unittest import TestCase | from unittest import TestCase | ||||
from swh.loader.core.tests import BaseLoaderTest, LoaderNoStorage | from swh.loader.core.tests import BaseLoaderTest | ||||
from swh.loader.svn.loader import (DEFAULT_BRANCH, SvnLoader, | from swh.loader.svn.loader import (DEFAULT_BRANCH, SvnLoader, | ||||
SvnLoaderFromRemoteDump, build_swh_snapshot) | SvnLoaderFromRemoteDump, build_swh_snapshot) | ||||
from swh.model import hashutil | from swh.model import hashutil | ||||
class BaseSvnLoaderTest(BaseLoaderTest): | |||||
"""Base test loader class. | |||||
In its setup, it's uncompressing a local svn mirror to /tmp. | |||||
""" | |||||
def setUp(self, archive_name='pkg-gourmet.tgz', filename='pkg-gourmet'): | |||||
super().setUp(archive_name=archive_name, filename=filename, | |||||
prefix_tmp_folder_name='swh.loader.svn.', | |||||
start_path=os.path.dirname(__file__)) | |||||
self.svn_mirror_url = self.repo_url | |||||
class TestSnapshot(TestCase): | class TestSnapshot(TestCase): | ||||
def test_build_swh_snapshot(self): | def test_build_swh_snapshot(self): | ||||
ardumont: What do you think of:
```
if not loader:
loader = SvnLoaderMemoryStorage()
self.loader =… | |||||
Not Done Inline Actionsself.loader = loader or SvnLoaderMemoryStorage() ;) vlorentz: ```
self.loader = loader or SvnLoaderMemoryStorage()
```
;) | |||||
Done Inline Actionsindeed, better anlambert: indeed, better | |||||
Not Done Inline Actions\m/ ardumont: \m/ | |||||
actual_snap = build_swh_snapshot('revision-id') | actual_snap = build_swh_snapshot('revision-id') | ||||
self.assertEqual(actual_snap, { | self.assertEqual(actual_snap, { | ||||
'id': None, | 'id': None, | ||||
'branches': { | 'branches': { | ||||
DEFAULT_BRANCH: { | DEFAULT_BRANCH: { | ||||
'target': 'revision-id', | 'target': 'revision-id', | ||||
'target_type': 'revision', | 'target_type': 'revision', | ||||
} | } | ||||
} | } | ||||
}) | }) | ||||
class LoaderWithState: | _LOADER_TEST_CONFIG = { | ||||
"""Additional state setup (bypassed by some override for test purposes) | 'check_revision': {'limit': 100, 'status': False}, | ||||
'content_packet_block_size_bytes': 104857600, | |||||
'content_packet_size': 10000, | |||||
'content_packet_size_bytes': 1073741824, | |||||
'content_size_limit': 104857600, | |||||
'debug': False, | |||||
'directory_packet_size': 2500, | |||||
'log_db': 'dbname=softwareheritage-log', | |||||
'occurrence_packet_size': 1000, | |||||
'release_packet_size': 1000, | |||||
'revision_packet_size': 10, | |||||
'save_data': False, | |||||
'save_data_path': '', | |||||
'send_contents': True, | |||||
'send_directories': True, | |||||
'send_occurrences': True, | |||||
'send_releases': True, | |||||
'send_revisions': True, | |||||
'send_snapshot': True, | |||||
'storage': {'args': {}, 'cls': 'memory'}, | |||||
'temp_directory': '/tmp' | |||||
} | |||||
class SvnLoaderTest(SvnLoader): | |||||
"""An SVNLoader with no persistence. | |||||
Context: | |||||
Load a new svn repository using the swh policy (so no update). | |||||
""" | """ | ||||
def __init__(self): | def __init__(self, last_snp_rev={}): | ||||
super().__init__() | super().__init__() | ||||
self.origin = { | self.origin = { | ||||
'id': 1, | 'id': 1, | ||||
'url': '/dev/null', | 'url': '/dev/null', | ||||
'type': 'svn', | 'type': 'svn', | ||||
} | } | ||||
self.visit = { | self.visit = { | ||||
'origin': 1, | 'origin': 1, | ||||
'visit': 1, | 'visit': 1, | ||||
} | } | ||||
self.last_snp_rev = last_snp_rev | |||||
def parse_config_file(self, *args, **kwargs): | |||||
return _LOADER_TEST_CONFIG | |||||
class SvnLoaderNoStorage(LoaderNoStorage, LoaderWithState, SvnLoader): | |||||
"""An SVNLoader with no persistence. | |||||
Context: | |||||
Load a new svn repository using the swh policy (so no update). | |||||
""" | |||||
def swh_latest_snapshot_revision(self, origin_id, prev_swh_revision=None): | |||||
"""We do not know this repository so no revision. | |||||
""" | |||||
return {} | |||||
class SvnLoaderUpdateNoStorage(LoaderNoStorage, LoaderWithState, SvnLoader): | |||||
"""An SVNLoader with no persistence. | |||||
Context: | |||||
Load a known svn repository using the swh policy. | |||||
We can either: | |||||
- do nothing since it does not contain any new commit (so no | |||||
change) | |||||
- either check its history is not altered and update in | |||||
consequence by loading the new revision | |||||
""" | |||||
def swh_latest_snapshot_revision(self, origin_id, prev_swh_revision=None): | def swh_latest_snapshot_revision(self, origin_id, prev_swh_revision=None): | ||||
Not Done Inline ActionsDo we still do it that way? I mean, IIRC, it was there to override the svn loader to bypass the real connection with the collaborator storage (no in-memory at the time). Now that we are using the memory storage, we supposedly can set the collaborator memory storage up to do its job (here to not have any snapshot to return for the specified origin in that particular case). But that can also be done later ;) Note: I won't repeat this everywhere but that remark/question stands for all different loader implementations below ;) ardumont: Do we still do it that way?
I mean, IIRC, it was there to override the svn loader to bypass… | |||||
Not Done Inline ActionsAgreed, all classes with "Storage" in their name should be dropped vlorentz: Agreed, all classes with "Storage" in their name should be dropped | |||||
Done Inline ActionsWill see what I can do anlambert: Will see what I can do | |||||
"""Avoid the storage persistence call and return the expected previous | """Avoid the storage persistence call and return the expected previous | ||||
revision for that repository. | revision for that repository. | ||||
Check the following for explanation about the hashes: | Check the following for explanation about the hashes: | ||||
- test_loader.org for (swh policy). | - test_loader.org for (swh policy). | ||||
- cf. SvnLoaderITTest | - cf. SvnLoaderTest | ||||
""" | """ | ||||
return { | return self.last_snp_rev | ||||
'snapshot': 'something', # need a snapshot of sort | |||||
'revision': { | |||||
'id': hashutil.hash_to_bytes( | |||||
'4876cb10aec6f708f7466dddf547567b65f6c39c'), | |||||
'parents': [hashutil.hash_to_bytes( | |||||
'a3a577948fdbda9d1061913b77a1588695eadb41')], | |||||
'directory': hashutil.hash_to_bytes( | |||||
'0deab3023ac59398ae467fc4bff5583008af1ee2'), | |||||
'target_type': 'revision', | |||||
'metadata': { | |||||
'extra_headers': [ | |||||
['svn_repo_uuid', | |||||
'3187e211-bb14-4c82-9596-0b59d67cd7f4'], | |||||
['svn_revision', '6'] | |||||
] | |||||
} | |||||
} | |||||
} | |||||
class SvnLoaderUpdateHistoryAlteredNoStorage(LoaderNoStorage, LoaderWithState, | |||||
SvnLoader): | |||||
"""Context: Load a known svn repository using the swh policy with its | |||||
history altered so we do not update it. | |||||
""" | class BaseSvnLoaderTest(BaseLoaderTest): | ||||
def swh_latest_snapshot_revision(self, origin_id, prev_swh_revision=None): | """Base test loader class. | ||||
"""Avoid the storage persistence call and return the expected previous | |||||
revision for that repository. | |||||
Check the following for explanation about the hashes: | In its setup, it's uncompressing a local svn mirror to /tmp. | ||||
- test_loader.org for (swh policy). | |||||
- cf. SvnLoaderITTest | |||||
""" | """ | ||||
return { | def setUp(self, archive_name='pkg-gourmet.tgz', filename='pkg-gourmet', | ||||
'snapshot': None, | loader=None): | ||||
'revision': { | super().setUp(archive_name=archive_name, filename=filename, | ||||
# Changed the revision id's hash to simulate history altered | prefix_tmp_folder_name='swh.loader.svn.', | ||||
'id': hashutil.hash_to_bytes( | start_path=os.path.dirname(__file__)) | ||||
'badbadbadbadf708f7466dddf547567b65f6c39d'), | self.svn_mirror_url = self.repo_url | ||||
'parents': [hashutil.hash_to_bytes( | self.loader = loader or SvnLoaderTest() | ||||
'a3a577948fdbda9d1061913b77a1588695eadb41')], | self.storage = self.loader.storage | ||||
'directory': hashutil.hash_to_bytes( | |||||
'0deab3023ac59398ae467fc4bff5583008af1ee2'), | |||||
'target_type': 'revision', | |||||
'metadata': { | |||||
'extra_headers': [ | |||||
['svn_repo_uuid', | |||||
'3187e211-bb14-4c82-9596-0b59d67cd7f4'], | |||||
['svn_revision', b'6'] | |||||
] | |||||
} | |||||
} | |||||
} | |||||
class SvnLoaderITest1(BaseSvnLoaderTest): | class SvnLoaderTest1(BaseSvnLoaderTest): | ||||
"""Load an unknown svn repository results in new data. | """Load an unknown svn repository results in new data. | ||||
""" | """ | ||||
def setUp(self): | |||||
super().setUp() | |||||
self.loader = SvnLoaderNoStorage() | |||||
def test_load(self): | def test_load(self): | ||||
"""Load a new repository results in new swh object and snapshot | """Load a new repository results in new swh object and snapshot | ||||
""" | """ | ||||
# when | # when | ||||
self.loader.load( | self.loader.load( | ||||
svn_url=self.svn_mirror_url, | svn_url=self.svn_mirror_url, | ||||
destination_path=self.destination_path) | destination_path=self.destination_path) | ||||
Show All 10 Lines | def test_load(self): | ||||
'0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71': '669a71cce6c424a81ba42b7dc5d560d32252f0ca', # noqa | '0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71': '669a71cce6c424a81ba42b7dc5d560d32252f0ca', # noqa | ||||
'95edacc8848369d6fb1608e887d6d2474fd5224f': '008ac97a1118560797c50e3392fa1443acdaa349', # noqa | '95edacc8848369d6fb1608e887d6d2474fd5224f': '008ac97a1118560797c50e3392fa1443acdaa349', # noqa | ||||
'fef26ea45a520071711ba2b9d16a2985ee837021': '3780effbe846a26751a95a8c95c511fb72be15b4', # noqa | 'fef26ea45a520071711ba2b9d16a2985ee837021': '3780effbe846a26751a95a8c95c511fb72be15b4', # noqa | ||||
'3f51abf3b3d466571be0855dfa67e094f9ceff1b': 'ffcca9b09c5827a6b8137322d4339c8055c3ee1e', # noqa | '3f51abf3b3d466571be0855dfa67e094f9ceff1b': 'ffcca9b09c5827a6b8137322d4339c8055c3ee1e', # noqa | ||||
'a3a577948fdbda9d1061913b77a1588695eadb41': '7dc52cc04c3b8bd7c085900d60c159f7b846f866', # noqa | 'a3a577948fdbda9d1061913b77a1588695eadb41': '7dc52cc04c3b8bd7c085900d60c159f7b846f866', # noqa | ||||
last_revision: '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa | last_revision: '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa | ||||
} | } | ||||
self.assertRevisionsOk(expected_revisions) | self.assertRevisionsContain(expected_revisions) | ||||
self.assertCountSnapshots(1) | self.assertCountSnapshots(1) | ||||
# FIXME: Check the snapshot's state | # FIXME: Check the snapshot's state | ||||
# self.assertEqual(self.loader.all_snapshots[0], {}) | # self.assertEqual(self.loader.all_snapshots[0], {}) | ||||
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | ||||
self.assertEqual(self.loader.visit_status(), 'full') | self.assertEqual(self.loader.visit_status(), 'full') | ||||
class SvnLoaderITest2(BaseSvnLoaderTest): | _LAST_SNP_REV = { | ||||
'snapshot': { | |||||
'id': 'something', | |||||
'branches': {} | |||||
}, # need a snapshot of sort | |||||
'revision': { | |||||
'id': hashutil.hash_to_bytes( | |||||
'4876cb10aec6f708f7466dddf547567b65f6c39c'), | |||||
'parents': [hashutil.hash_to_bytes( | |||||
'a3a577948fdbda9d1061913b77a1588695eadb41')], | |||||
'directory': hashutil.hash_to_bytes( | |||||
'0deab3023ac59398ae467fc4bff5583008af1ee2'), | |||||
'target_type': 'revision', | |||||
'metadata': { | |||||
'extra_headers': [ | |||||
['svn_repo_uuid', | |||||
'3187e211-bb14-4c82-9596-0b59d67cd7f4'], | |||||
['svn_revision', '6'] | |||||
] | |||||
} | |||||
} | |||||
} | |||||
class SvnLoaderTest2(BaseSvnLoaderTest): | |||||
Not Done Inline Actionsrename in caps lock (that's a constant). Also probably split snapshot and revision. vlorentz: rename in caps lock (that's a constant). Also probably split snapshot and revision. | |||||
Done Inline Actionsack anlambert: ack | |||||
"""Load a visited repository with no new change results in no data | """Load a visited repository with no new change results in no data | ||||
change. | change. | ||||
""" | """ | ||||
def setUp(self): | def setUp(self): | ||||
super().setUp() | super().setUp(loader=SvnLoaderTest(last_snp_rev=_LAST_SNP_REV)) | ||||
Not Done Inline ActionsReplace with: super().setUp(loader=SvnLoaderTest) self.storage.revision_add([_last_snp_rev['revision']]) self.storage.snapshot_add([_last_snp_rev['snapshot']]) And remove/rename all occurences of last_snp_rev. vlorentz: Replace with:
```
super().setUp(loader=SvnLoaderTest)
self.storage.revision_add([_last_snp_rev… | |||||
Done Inline ActionsUnfortunately, that's not so simple. I have just tested to remove the swh_latest_snapshot_revision override in the test loader class in favor of only relying to the in-memory storage but numerous errors appeared. So I will keep it this way for the moment. anlambert: Unfortunately, that's not so simple. I have just tested to remove the… | |||||
self.loader = SvnLoaderUpdateNoStorage() | |||||
def test_load(self): | def test_load(self): | ||||
"""Load a repository without new changes results in same snapshot | """Load a repository without new changes results in same snapshot | ||||
""" | """ | ||||
# when | # when | ||||
self.loader.load( | self.loader.load( | ||||
svn_url=self.svn_mirror_url, | svn_url=self.svn_mirror_url, | ||||
destination_path=self.destination_path) | destination_path=self.destination_path) | ||||
# then | # then | ||||
self.assertCountContents(0) | self.assertCountContents(0) | ||||
self.assertCountDirectories(0) | self.assertCountDirectories(0) | ||||
self.assertCountRevisions(0) | self.assertCountRevisions(0) | ||||
self.assertCountReleases(0) | self.assertCountReleases(0) | ||||
self.assertCountSnapshots(1) | self.assertCountSnapshots(1) | ||||
# FIXME: Check the snapshot's state | # FIXME: Check the snapshot's state | ||||
# self.assertEqual(self.loader.all_snapshots[0], {}) | # self.assertEqual(self.loader.all_snapshots[0], {}) | ||||
self.assertEqual(self.loader.load_status(), {'status': 'uneventful'}) | self.assertEqual(self.loader.load_status(), {'status': 'uneventful'}) | ||||
self.assertEqual(self.loader.visit_status(), 'full') | self.assertEqual(self.loader.visit_status(), 'full') | ||||
class SvnLoaderITest3(BaseSvnLoaderTest): | class SvnLoaderTest3(BaseSvnLoaderTest): | ||||
"""In this scenario, the dump has been tampered with to modify the | """In this scenario, the dump has been tampered with to modify the | ||||
commit log. This results in a hash divergence which is | commit log. This results in a hash divergence which is | ||||
detected at startup. | detected at startup. | ||||
In effect, that stops the loading and do nothing. | In effect, that stops the loading and do nothing. | ||||
""" | """ | ||||
def setUp(self): | def setUp(self): | ||||
last_snp_rev = copy.deepcopy(_LAST_SNP_REV) | |||||
last_snp_rev['snapshot'] = None | |||||
# Changed the revision id's hash to simulate history altered | |||||
last_snp_rev['revision']['id'] = \ | |||||
hashutil.hash_to_bytes('badbadbadbadf708f7466dddf547567b65f6c39d') | |||||
# the svn repository pkg-gourmet has been updated with changes | # the svn repository pkg-gourmet has been updated with changes | ||||
super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | super().setUp(archive_name='pkg-gourmet-with-updates.tgz', | ||||
self.loader = SvnLoaderUpdateHistoryAlteredNoStorage() | loader=SvnLoaderTest(last_snp_rev=last_snp_rev)) | ||||
def test_load(self): | def test_load(self): | ||||
"""Load known repository with history altered should do nothing | """Load known repository with history altered should do nothing | ||||
""" | """ | ||||
# when | # when | ||||
self.loader.load(svn_url=self.svn_mirror_url, | self.loader.load(svn_url=self.svn_mirror_url, | ||||
destination_path=self.destination_path) | destination_path=self.destination_path) | ||||
# then | # then | ||||
# we got the previous run's last revision (rev 6) | # we got the previous run's last revision (rev 6) | ||||
Not Done Inline Actionskind of a duplicate of the global one. vlorentz: kind of a duplicate of the global one. | |||||
Done Inline Actionsack anlambert: ack | |||||
# so 2 news + 1 old | # so 2 news + 1 old | ||||
self.assertCountContents(0) | self.assertCountContents(0) | ||||
self.assertCountDirectories(0) | self.assertCountDirectories(0) | ||||
self.assertCountRevisions(0) | self.assertCountRevisions(0) | ||||
self.assertCountReleases(0) | self.assertCountReleases(0) | ||||
self.assertCountSnapshots(0) | self.assertCountSnapshots(0) | ||||
# FIXME: Check the snapshot's state | # FIXME: Check the snapshot's state | ||||
# self.assertEqual(self.loader.all_snapshots[0], {}) | # self.assertEqual(self.loader.all_snapshots[0], {}) | ||||
self.assertEqual(self.loader.load_status(), {'status': 'uneventful'}) | self.assertEqual(self.loader.load_status(), {'status': 'uneventful'}) | ||||
self.assertEqual(self.loader.visit_status(), 'partial') | self.assertEqual(self.loader.visit_status(), 'partial') | ||||
class SvnLoaderITest4(BaseSvnLoaderTest): | class SvnLoaderTest4(BaseSvnLoaderTest): | ||||
"""In this scenario, the repository has been updated with new changes. | """In this scenario, the repository has been updated with new changes. | ||||
The loading visit should result in new objects stored and 1 new | The loading visit should result in new objects stored and 1 new | ||||
snapshot. | snapshot. | ||||
""" | """ | ||||
def setUp(self): | def setUp(self): | ||||
# the svn repository pkg-gourmet has been updated with changes | # the svn repository pkg-gourmet has been updated with changes | ||||
super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | super().setUp(archive_name='pkg-gourmet-with-updates.tgz', | ||||
self.loader = SvnLoaderUpdateNoStorage() | loader=SvnLoaderTest(last_snp_rev=_LAST_SNP_REV)) | ||||
def test_process_repository(self): | def test_process_repository(self): | ||||
"""Process updated repository should yield new objects | """Process updated repository should yield new objects | ||||
""" | """ | ||||
# when | # when | ||||
self.loader.load(svn_url=self.svn_mirror_url, | self.loader.load(svn_url=self.svn_mirror_url, | ||||
destination_path=self.destination_path) | destination_path=self.destination_path) | ||||
Show All 11 Lines | def test_process_repository(self): | ||||
# revision hash | directory hash | # revision hash | directory hash | ||||
'7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa | '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa | ||||
'38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa | '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa | ||||
'99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa | '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa | ||||
'902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa | '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa | ||||
last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa | last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa | ||||
} | } | ||||
self.assertRevisionsOk(expected_revisions) | self.assertRevisionsContain(expected_revisions) | ||||
self.assertCountSnapshots(1) | self.assertCountSnapshots(1) | ||||
# FIXME: Check the snapshot's state | # FIXME: Check the snapshot's state | ||||
# self.assertEqual(self.loader.all_snapshots[0], {}) | # self.assertEqual(self.loader.all_snapshots[0], {}) | ||||
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | ||||
self.assertEqual(self.loader.visit_status(), 'full') | self.assertEqual(self.loader.visit_status(), 'full') | ||||
class SvnLoaderITTest5(BaseSvnLoaderTest): | class SvnLoaderTest5(BaseSvnLoaderTest): | ||||
"""Context: | """Context: | ||||
- Repository already injected with successful data | - Repository already injected with successful data | ||||
- New visit from scratch done with successful load | - New visit from scratch done with successful load | ||||
""" | """ | ||||
def setUp(self): | def setUp(self): | ||||
# the svn repository pkg-gourmet has been updated with changes | # the svn repository pkg-gourmet has been updated with changes | ||||
super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | super().setUp(archive_name='pkg-gourmet-with-updates.tgz', | ||||
self.loader = SvnLoaderUpdateNoStorage() | loader=SvnLoaderTest(last_snp_rev=_LAST_SNP_REV)) | ||||
def test_load(self): | def test_load(self): | ||||
"""Load an existing repository from scratch yields same swh objects | """Load an existing repository from scratch yields same swh objects | ||||
""" | """ | ||||
# when | # when | ||||
self.loader.load(svn_url=self.svn_mirror_url, | self.loader.load(svn_url=self.svn_mirror_url, | ||||
destination_path=self.destination_path, | destination_path=self.destination_path, | ||||
Show All 15 Lines | def test_load(self): | ||||
'4876cb10aec6f708f7466dddf547567b65f6c39c': '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa | '4876cb10aec6f708f7466dddf547567b65f6c39c': '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa | ||||
'7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa | '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa | ||||
'38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa | '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa | ||||
'99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa | '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa | ||||
'902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa | '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa | ||||
'171dc35522bfd17dda4e90a542a0377fb2fc707a': 'fd24a76c87a3207428e06612b49860fc78e9f6dc', # noqa | '171dc35522bfd17dda4e90a542a0377fb2fc707a': 'fd24a76c87a3207428e06612b49860fc78e9f6dc', # noqa | ||||
} | } | ||||
self.assertRevisionsOk(expected_revisions) | self.assertRevisionsContain(expected_revisions) | ||||
self.assertCountSnapshots(1) | self.assertCountSnapshots(1) | ||||
# FIXME: Check the snapshot's state | # FIXME: Check the snapshot's state | ||||
# self.assertEqual(self.loader.all_snapshots[0], {}) | # self.assertEqual(self.loader.all_snapshots[0], {}) | ||||
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | ||||
self.assertEqual(self.loader.visit_status(), 'full') | self.assertEqual(self.loader.visit_status(), 'full') | ||||
class SvnLoaderWithPreviousRevisionNoStorage(LoaderNoStorage, LoaderWithState, | class SvnLoaderTest6(BaseSvnLoaderTest): | ||||
SvnLoader): | """Context: | ||||
"""An SVNLoader with no persistence. | - repository already visited with load successful | ||||
- Changes on existing repository | |||||
Context: Load a known svn repository using the swh policy with its | - New Visit done with successful new data | ||||
history altered so we do not update it. | |||||
""" | |||||
def swh_latest_snapshot_revision(self, origin_id, prev_swh_revision=None): | |||||
"""Avoid the storage persistence call and return the expected previous | |||||
revision for that repository. | |||||
Check the following for explanation about the hashes: | |||||
- test_loader.org for (swh policy). | |||||
- cf. SvnLoaderITTest | |||||
""" | """ | ||||
return { | def setUp(self): | ||||
last_snp_rev = { | |||||
'snapshot': None, | 'snapshot': None, | ||||
'revision': { | 'revision': { | ||||
'id': hashutil.hash_to_bytes( | 'id': hashutil.hash_to_bytes( | ||||
'4876cb10aec6f708f7466dddf547567b65f6c39c'), | '4876cb10aec6f708f7466dddf547567b65f6c39c'), | ||||
'parents': [hashutil.hash_to_bytes( | 'parents': [hashutil.hash_to_bytes( | ||||
'a3a577948fdbda9d1061913b77a1588695eadb41')], | 'a3a577948fdbda9d1061913b77a1588695eadb41')], | ||||
'directory': hashutil.hash_to_bytes( | 'directory': hashutil.hash_to_bytes( | ||||
'0deab3023ac59398ae467fc4bff5583008af1ee2'), | '0deab3023ac59398ae467fc4bff5583008af1ee2'), | ||||
'target_type': 'revision', | 'target_type': 'revision', | ||||
'metadata': { | 'metadata': { | ||||
'extra_headers': [ | 'extra_headers': [ | ||||
['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], # noqa | ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], # noqa | ||||
['svn_revision', '6'] | ['svn_revision', '6'] | ||||
] | ] | ||||
} | } | ||||
} | } | ||||
} | } | ||||
super().setUp(archive_name='pkg-gourmet-with-updates.tgz', | |||||
loader=SvnLoaderTest(last_snp_rev=last_snp_rev)) | |||||
class SvnLoaderITTest6(BaseSvnLoaderTest): | |||||
"""Context: | |||||
- repository already visited with load successful | |||||
- Changes on existing repository | |||||
- New Visit done with successful new data | |||||
""" | |||||
def setUp(self): | |||||
super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | |||||
self.loader = SvnLoaderWithPreviousRevisionNoStorage() | |||||
def test_load(self): | def test_load(self): | ||||
"""Load from partial previous visit result in new changes | """Load from partial previous visit result in new changes | ||||
""" | """ | ||||
# when | # when | ||||
self.loader.load( | self.loader.load( | ||||
svn_url=self.svn_mirror_url, | svn_url=self.svn_mirror_url, | ||||
Show All 12 Lines | def test_load(self): | ||||
# revision hash | directory hash | # revision hash | directory hash | ||||
'7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa | '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa | ||||
'38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa | '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa | ||||
'99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa | '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa | ||||
'902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa | '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa | ||||
last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa | last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa | ||||
} | } | ||||
self.assertRevisionsOk(expected_revisions) | self.assertRevisionsContain(expected_revisions) | ||||
self.assertCountSnapshots(1) | self.assertCountSnapshots(1) | ||||
# FIXME: Check the snapshot's state | # FIXME: Check the snapshot's state | ||||
# self.assertEqual(self.loader.all_snapshots[0], {}) | # self.assertEqual(self.loader.all_snapshots[0], {}) | ||||
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | ||||
self.assertEqual(self.loader.visit_status(), 'full') | self.assertEqual(self.loader.visit_status(), 'full') | ||||
class SvnLoaderITest7(BaseSvnLoaderTest): | class SvnLoaderTest7(BaseSvnLoaderTest): | ||||
"""Context: | """Context: | ||||
- repository already visited with load successful | - repository already visited with load successful | ||||
- Changes on existing repository | - Changes on existing repository | ||||
- New Visit done with successful new data | - New Visit done with successful new data | ||||
""" | """ | ||||
def setUp(self): | def setUp(self): | ||||
super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | super().setUp(archive_name='pkg-gourmet-with-updates.tgz', | ||||
self.loader = SvnLoaderUpdateNoStorage() | loader=SvnLoaderTest(last_snp_rev=_LAST_SNP_REV)) | ||||
def test_load(self): | def test_load(self): | ||||
"""Load known and partial repository should start from last visit | """Load known and partial repository should start from last visit | ||||
""" | """ | ||||
previous_unfinished_revision = { | previous_unfinished_revision = { | ||||
'id': hashutil.hash_to_bytes( | 'id': hashutil.hash_to_bytes( | ||||
'a3a577948fdbda9d1061913b77a1588695eadb41'), | 'a3a577948fdbda9d1061913b77a1588695eadb41'), | ||||
Show All 29 Lines | def test_load(self): | ||||
# revision hash | directory hash | # revision hash | directory hash | ||||
'7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa | '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa | ||||
'38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa | '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa | ||||
'99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa | '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa | ||||
'902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa | '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa | ||||
last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa | last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa | ||||
} | } | ||||
self.assertRevisionsOk(expected_revisions) | self.assertRevisionsContain(expected_revisions) | ||||
self.assertCountSnapshots(1) | self.assertCountSnapshots(1) | ||||
# FIXME: Check the snapshot's state | # FIXME: Check the snapshot's state | ||||
# self.assertEqual(self.loader.all_snapshots[0], {}) | # self.assertEqual(self.loader.all_snapshots[0], {}) | ||||
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | ||||
self.assertEqual(self.loader.visit_status(), 'full') | self.assertEqual(self.loader.visit_status(), 'full') | ||||
class SvnLoaderUpdateLessRecentNoStorage(LoaderNoStorage, LoaderWithState, | class SvnLoaderTest8(BaseSvnLoaderTest): | ||||
SvnLoader): | |||||
"""Context: | """Context: | ||||
Load a known svn repository. The last visit seen is less | |||||
recent than a previous unfinished crawl. | |||||
""" | |||||
def swh_latest_snapshot_revision(self, origin_id, prev_swh_revision=None): | |||||
"""Avoid the storage persistence call and return the expected previous | |||||
revision for that repository. | |||||
Check the following for explanation about the hashes: | - Previous visit on existing repository done | ||||
- test_loader.org for (swh policy). | - Starting the loading from the last unfinished visit | ||||
- cf. SvnLoaderITTest | - New objects are created (1 new snapshot) | ||||
""" | """ | ||||
return { | def setUp(self): | ||||
last_snp_rev = { | |||||
'snapshot': None, | 'snapshot': None, | ||||
'revision': { | 'revision': { | ||||
'id': hashutil.hash_to_bytes( | 'id': hashutil.hash_to_bytes( | ||||
'a3a577948fdbda9d1061913b77a1588695eadb41'), | 'a3a577948fdbda9d1061913b77a1588695eadb41'), | ||||
'parents': [hashutil.hash_to_bytes( | 'parents': [hashutil.hash_to_bytes( | ||||
'3f51abf3b3d466571be0855dfa67e094f9ceff1b')], | '3f51abf3b3d466571be0855dfa67e094f9ceff1b')], | ||||
'directory': hashutil.hash_to_bytes( | 'directory': hashutil.hash_to_bytes( | ||||
'7dc52cc04c3b8bd7c085900d60c159f7b846f866'), | '7dc52cc04c3b8bd7c085900d60c159f7b846f866'), | ||||
'target_type': 'revision', | 'target_type': 'revision', | ||||
'metadata': { | 'metadata': { | ||||
'extra_headers': [ | 'extra_headers': [ | ||||
['svn_repo_uuid', | ['svn_repo_uuid', | ||||
'3187e211-bb14-4c82-9596-0b59d67cd7f4'], | '3187e211-bb14-4c82-9596-0b59d67cd7f4'], | ||||
['svn_revision', '5'] | ['svn_revision', '5'] | ||||
] | ] | ||||
} | } | ||||
} | } | ||||
} | } | ||||
super().setUp(archive_name='pkg-gourmet-with-updates.tgz', | |||||
loader=SvnLoaderTest(last_snp_rev=last_snp_rev)) | |||||
class SvnLoaderITest8(BaseSvnLoaderTest): | |||||
"""Context: | |||||
- Previous visit on existing repository done | |||||
- Starting the loading from the last unfinished visit | |||||
- New objects are created (1 new snapshot) | |||||
""" | |||||
def setUp(self): | |||||
super().setUp(archive_name='pkg-gourmet-with-updates.tgz') | |||||
self.loader = SvnLoaderUpdateLessRecentNoStorage() | |||||
def test_load(self): | def test_load(self): | ||||
"""Load repository should yield revisions starting from last visit | """Load repository should yield revisions starting from last visit | ||||
""" | """ | ||||
previous_unfinished_revision = { | previous_unfinished_revision = { | ||||
'id': hashutil.hash_to_bytes( | 'id': hashutil.hash_to_bytes( | ||||
'4876cb10aec6f708f7466dddf547567b65f6c39c'), | '4876cb10aec6f708f7466dddf547567b65f6c39c'), | ||||
Show All 28 Lines | def test_load(self): | ||||
# revision hash | directory hash | # revision hash | directory hash | ||||
'7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa | '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa | ||||
'38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa | '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa | ||||
'99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa | '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa | ||||
'902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa | '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa | ||||
last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa | last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa | ||||
} | } | ||||
self.assertRevisionsOk(expected_revisions) | self.assertRevisionsContain(expected_revisions) | ||||
self.assertCountSnapshots(1) | self.assertCountSnapshots(1) | ||||
# FIXME: Check the snapshot's state | # FIXME: Check the snapshot's state | ||||
# self.assertEqual(self.loader.all_snapshots[0], {}) | # self.assertEqual(self.loader.all_snapshots[0], {}) | ||||
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | ||||
self.assertEqual(self.loader.visit_status(), 'full') | self.assertEqual(self.loader.visit_status(), 'full') | ||||
class SvnLoaderTTest9(BaseSvnLoaderTest): | class SvnLoaderTTest9(BaseSvnLoaderTest): | ||||
"""Check that a svn repo containing a versioned file with CRLF line | """Check that a svn repo containing a versioned file with CRLF line | ||||
endings with svn:eol-style property set to 'native' (this is a | endings with svn:eol-style property set to 'native' (this is a | ||||
violation of svn specification as the file should have been | violation of svn specification as the file should have been | ||||
stored with LF line endings) can be loaded anyway. | stored with LF line endings) can be loaded anyway. | ||||
""" | """ | ||||
def setUp(self): | def setUp(self): | ||||
super().setUp(archive_name='mediawiki-repo-r407-eol-native-crlf.tgz', | super().setUp(archive_name='mediawiki-repo-r407-eol-native-crlf.tgz', | ||||
filename='mediawiki-repo-r407-eol-native-crlf') | filename='mediawiki-repo-r407-eol-native-crlf') | ||||
self.loader = SvnLoaderNoStorage() | |||||
def test_process_repository(self): | def test_process_repository(self): | ||||
"""Load repository with CRLF endings (svn:eol-style: native) is ok | """Load repository with CRLF endings (svn:eol-style: native) is ok | ||||
""" # noqa | """ # noqa | ||||
# when | # when | ||||
self.loader.load(svn_url=self.svn_mirror_url, | self.loader.load(svn_url=self.svn_mirror_url, | ||||
destination_path=self.destination_path) | destination_path=self.destination_path) | ||||
expected_revisions = { | expected_revisions = { | ||||
'7da4975c363101b819756d33459f30a866d01b1b': 'f63637223ee0f7d4951ffd2d4d9547a4882c5d8b' # noqa | '7da4975c363101b819756d33459f30a866d01b1b': 'f63637223ee0f7d4951ffd2d4d9547a4882c5d8b' # noqa | ||||
} | } | ||||
self.assertRevisionsOk(expected_revisions) | self.assertRevisionsContain(expected_revisions) | ||||
self.assertCountSnapshots(1) | self.assertCountSnapshots(1) | ||||
# FIXME: Check the snapshot's state | # FIXME: Check the snapshot's state | ||||
# self.assertEqual(self.loader.all_snapshots[0], {}) | # self.assertEqual(self.loader.all_snapshots[0], {}) | ||||
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | ||||
self.assertEqual(self.loader.visit_status(), 'full') | self.assertEqual(self.loader.visit_status(), 'full') | ||||
class SvnLoaderITest10(BaseSvnLoaderTest): # noqa | class SvnLoaderTest10(BaseSvnLoaderTest): # noqa | ||||
"""Check that a svn repo containing a versioned file with mixed | """Check that a svn repo containing a versioned file with mixed | ||||
CRLF/LF line endings with svn:eol-style property set to 'native' | CRLF/LF line endings with svn:eol-style property set to 'native' | ||||
(this is a violation of svn specification as mixed line endings | (this is a violation of svn specification as mixed line endings | ||||
for textual content should not be stored when the svn:eol-style | for textual content should not be stored when the svn:eol-style | ||||
property is set) can be loaded anyway. | property is set) can be loaded anyway. | ||||
""" | """ | ||||
def setUp(self): | def setUp(self): | ||||
super().setUp( | super().setUp( | ||||
archive_name='pyang-repo-r343-eol-native-mixed-lf-crlf.tgz', | archive_name='pyang-repo-r343-eol-native-mixed-lf-crlf.tgz', | ||||
filename='pyang-repo-r343-eol-native-mixed-lf-crlf') | filename='pyang-repo-r343-eol-native-mixed-lf-crlf') | ||||
self.loader = SvnLoaderNoStorage() | |||||
def test_load(self): | def test_load(self): | ||||
"""Load repo with mixed CRLF/LF endings (svn:eol-style:native) is ok | """Load repo with mixed CRLF/LF endings (svn:eol-style:native) is ok | ||||
""" | """ | ||||
self.loader.load(svn_url=self.svn_mirror_url, | self.loader.load(svn_url=self.svn_mirror_url, | ||||
destination_path=self.destination_path) | destination_path=self.destination_path) | ||||
expected_revisions = { | expected_revisions = { | ||||
'9c6962eeb9164a636c374be700672355e34a98a7': '16aa6b6271f3456d4643999d234cf39fe3d0cc5a' # noqa | '9c6962eeb9164a636c374be700672355e34a98a7': '16aa6b6271f3456d4643999d234cf39fe3d0cc5a' # noqa | ||||
} | } | ||||
self.assertRevisionsOk(expected_revisions) | self.assertRevisionsContain(expected_revisions) | ||||
self.assertCountSnapshots(1) | self.assertCountSnapshots(1) | ||||
# FIXME: Check the snapshot's state | # FIXME: Check the snapshot's state | ||||
# self.assertEqual(self.loader.all_snapshots[0], {}) | # self.assertEqual(self.loader.all_snapshots[0], {}) | ||||
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | ||||
self.assertEqual(self.loader.visit_status(), 'full') | self.assertEqual(self.loader.visit_status(), 'full') | ||||
class SvnLoaderITest11(BaseSvnLoaderTest): | class SvnLoaderTest11(BaseSvnLoaderTest): | ||||
"""Context: | """Context: | ||||
- Repository with svn:external (which is not deal with for now) | - Repository with svn:external (which is not deal with for now) | ||||
- Visit is partial with as much data loaded as possible | - Visit is partial with as much data loaded as possible | ||||
""" | """ | ||||
def setUp(self): | def setUp(self): | ||||
super().setUp(archive_name='pkg-gourmet-with-external-id.tgz') | super().setUp(archive_name='pkg-gourmet-with-external-id.tgz') | ||||
self.loader = SvnLoaderNoStorage() | |||||
def test_load(self): | def test_load(self): | ||||
"""Repository with svn:externals property, will stop raising an error | """Repository with svn:externals property, will stop raising an error | ||||
""" | """ | ||||
previous_unfinished_revision = None | previous_unfinished_revision = None | ||||
# when | # when | ||||
Show All 29 Lines | def test_load(self): | ||||
'ded78810401fd354ffe894aa4a1e5c7d30a645d1': 'b0a648b02e55a4dce356ac35187a058f89694ec7', # noqa | 'ded78810401fd354ffe894aa4a1e5c7d30a645d1': 'b0a648b02e55a4dce356ac35187a058f89694ec7', # noqa | ||||
'4ee95e39358712f53c4fc720da3fafee9249ed19': 'c3c98df624733fef4e592bef983f93e2ed02b179', # noqa | '4ee95e39358712f53c4fc720da3fafee9249ed19': 'c3c98df624733fef4e592bef983f93e2ed02b179', # noqa | ||||
'ffa901b69ca0f46a2261f42948838d19709cb9f8': 'c3c98df624733fef4e592bef983f93e2ed02b179', # noqa | 'ffa901b69ca0f46a2261f42948838d19709cb9f8': 'c3c98df624733fef4e592bef983f93e2ed02b179', # noqa | ||||
'0148ae3eaa520b73a50802c59f3f416b7a36cf8c': '844d4646d6c2b4f3a3b2b22ab0ee38c7df07bab2', # noqa | '0148ae3eaa520b73a50802c59f3f416b7a36cf8c': '844d4646d6c2b4f3a3b2b22ab0ee38c7df07bab2', # noqa | ||||
last_revision: '0de6e75d2b79ec90d00a3a7611aa3861b2e4aa5e', # noqa | last_revision: '0de6e75d2b79ec90d00a3a7611aa3861b2e4aa5e', # noqa | ||||
} | } | ||||
# The last revision being the one used later to start back from | # The last revision being the one used later to start back from | ||||
self.assertRevisionsOk(expected_revisions) | self.assertRevisionsContain(expected_revisions) | ||||
self.assertCountSnapshots(1) | self.assertCountSnapshots(1) | ||||
# FIXME: Check the snapshot's state | # FIXME: Check the snapshot's state | ||||
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | ||||
self.assertEqual(self.loader.visit_status(), 'partial') | self.assertEqual(self.loader.visit_status(), 'partial') | ||||
class SvnLoaderITest12(BaseSvnLoaderTest): | class SvnLoaderTest12(BaseSvnLoaderTest): | ||||
"""Edge cases: | """Edge cases: | ||||
- first create a file and commit it. | - first create a file and commit it. | ||||
Remove it, then add folder holding the same name, commit. | Remove it, then add folder holding the same name, commit. | ||||
- do the same scenario with symbolic link (instead of file) | - do the same scenario with symbolic link (instead of file) | ||||
""" | """ | ||||
def setUp(self): | def setUp(self): | ||||
super().setUp( | super().setUp( | ||||
archive_name='pkg-gourmet-with-edge-case-links-and-files.tgz') | archive_name='pkg-gourmet-with-edge-case-links-and-files.tgz') | ||||
self.loader = SvnLoaderNoStorage() | |||||
def test_load(self): | def test_load(self): | ||||
"""File/Link removed prior to folder with same name creation is ok | """File/Link removed prior to folder with same name creation is ok | ||||
""" | """ | ||||
previous_unfinished_revision = None | previous_unfinished_revision = None | ||||
# when | # when | ||||
Show All 25 Lines | def test_load(self): | ||||
'bb78300cc1ac9119eb6fffa9e9fa04a7f9340b11': 'ee995a0d85f6917c75bcee3aa448bea7726b265d', # noqa | 'bb78300cc1ac9119eb6fffa9e9fa04a7f9340b11': 'ee995a0d85f6917c75bcee3aa448bea7726b265d', # noqa | ||||
'f2e01111329f84580dc3febb1fd45515692c5886': 'e2baec7b6a5543758e9c73695bc847db0a4f7941', # noqa | 'f2e01111329f84580dc3febb1fd45515692c5886': 'e2baec7b6a5543758e9c73695bc847db0a4f7941', # noqa | ||||
'1a0f70c34e211f073e1be3435ecf6f0dd7700267': 'e7536e721fa806c19971b749c091c144b2f2b88e', # noqa | '1a0f70c34e211f073e1be3435ecf6f0dd7700267': 'e7536e721fa806c19971b749c091c144b2f2b88e', # noqa | ||||
'0c612a23d293cc3100496a54ae4ad13d750efe4c': '2123d12749294bbfb54e73f9d73fac658aabb266', # noqa | '0c612a23d293cc3100496a54ae4ad13d750efe4c': '2123d12749294bbfb54e73f9d73fac658aabb266', # noqa | ||||
'69a53d972e2f863acbbbda546d9da96287af6a88': '13896cb96ec004140ce5be8852fee8c29830d9c7', # noqa | '69a53d972e2f863acbbbda546d9da96287af6a88': '13896cb96ec004140ce5be8852fee8c29830d9c7', # noqa | ||||
last_revision: '6b1e0243768ff9ac060064b2eeade77e764ffc82', # noqa | last_revision: '6b1e0243768ff9ac060064b2eeade77e764ffc82', # noqa | ||||
} | } | ||||
self.assertRevisionsOk(expected_revisions) | self.assertRevisionsContain(expected_revisions) | ||||
self.assertCountSnapshots(1) | self.assertCountSnapshots(1) | ||||
# FIXME: Check the snapshot's state | # FIXME: Check the snapshot's state | ||||
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | ||||
self.assertEqual(self.loader.visit_status(), 'full') | self.assertEqual(self.loader.visit_status(), 'full') | ||||
class SvnLoaderITTest13(BaseSvnLoaderTest): | class SvnLoaderTest13(BaseSvnLoaderTest): | ||||
"""Edge cases: | """Edge cases: | ||||
- wrong symbolic link | - wrong symbolic link | ||||
- wrong symbolic link with empty space names | - wrong symbolic link with empty space names | ||||
""" | """ | ||||
def setUp(self): | def setUp(self): | ||||
super().setUp( | super().setUp( | ||||
archive_name='pkg-gourmet-with-wrong-link-cases.tgz') | archive_name='pkg-gourmet-with-wrong-link-cases.tgz') | ||||
self.loader = SvnLoaderNoStorage() | |||||
def test_load(self): | def test_load(self): | ||||
"""Wrong link or empty space-named link should be ok | """Wrong link or empty space-named link should be ok | ||||
""" | """ | ||||
# when | # when | ||||
self.loader.load( | self.loader.load( | ||||
svn_url=self.svn_mirror_url, | svn_url=self.svn_mirror_url, | ||||
Show All 25 Lines | def test_load(self): | ||||
'1a0f70c34e211f073e1be3435ecf6f0dd7700267': 'e7536e721fa806c19971b749c091c144b2f2b88e', # noqa | '1a0f70c34e211f073e1be3435ecf6f0dd7700267': 'e7536e721fa806c19971b749c091c144b2f2b88e', # noqa | ||||
'0c612a23d293cc3100496a54ae4ad13d750efe4c': '2123d12749294bbfb54e73f9d73fac658aabb266', # noqa | '0c612a23d293cc3100496a54ae4ad13d750efe4c': '2123d12749294bbfb54e73f9d73fac658aabb266', # noqa | ||||
'69a53d972e2f863acbbbda546d9da96287af6a88': '13896cb96ec004140ce5be8852fee8c29830d9c7', # noqa | '69a53d972e2f863acbbbda546d9da96287af6a88': '13896cb96ec004140ce5be8852fee8c29830d9c7', # noqa | ||||
'3f43af2578fccf18b0d4198e48563da7929dc608': '6b1e0243768ff9ac060064b2eeade77e764ffc82', # noqa | '3f43af2578fccf18b0d4198e48563da7929dc608': '6b1e0243768ff9ac060064b2eeade77e764ffc82', # noqa | ||||
'4ab5fc264732cd474d2e695c5ac66e4933bdad74': '9a1f5e3961db89422250ce6c1441476f40d65205', # noqa | '4ab5fc264732cd474d2e695c5ac66e4933bdad74': '9a1f5e3961db89422250ce6c1441476f40d65205', # noqa | ||||
last_revision: 'd853d9628f6f0008d324fed27dadad00ce48bc62', # noqa | last_revision: 'd853d9628f6f0008d324fed27dadad00ce48bc62', # noqa | ||||
} | } | ||||
self.assertRevisionsOk(expected_revisions) | self.assertRevisionsContain(expected_revisions) | ||||
self.assertCountSnapshots(1) | self.assertCountSnapshots(1) | ||||
# FIXME: Check the snapshot's state | # FIXME: Check the snapshot's state | ||||
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | ||||
self.assertEqual(self.loader.visit_status(), 'full') | self.assertEqual(self.loader.visit_status(), 'full') | ||||
class SvnLoaderFromRemoteDumpNoStorage(LoaderNoStorage, LoaderWithState, | class SvnLoaderTestFromRemoteDump(SvnLoaderTest, SvnLoaderFromRemoteDump): | ||||
SvnLoaderFromRemoteDump): | pass | ||||
"""A SvnLoaderFromRemoteDump with no persistence. | |||||
Context: | |||||
Load a remote svn repository from a generated dump file. | |||||
""" | |||||
def swh_latest_snapshot_revision(self, origin_id, prev_swh_revision=None): | |||||
"""We do not know this repository so no revision. | |||||
""" | |||||
return {} | |||||
class SvnLoaderFromRemoteDump(BaseSvnLoaderTest): | class SvnLoaderFromRemoteDumpTest(BaseSvnLoaderTest): | ||||
""" | """ | ||||
Check that the results obtained with the remote svn dump loader | Check that the results obtained with the remote svn dump loader | ||||
and the base svn loader are the same. | and the base svn loader are the same. | ||||
""" | """ | ||||
def setUp(self): | def setUp(self): | ||||
super().setUp(archive_name='pkg-gourmet.tgz') | super().setUp(archive_name='pkg-gourmet.tgz', | ||||
loader=SvnLoaderTestFromRemoteDump()) | |||||
def test_load(self): | def test_load(self): | ||||
""" | """ | ||||
Compare results of remote dump loader and base loader | Compare results of remote dump loader and base loader | ||||
""" | """ | ||||
dump_loader = SvnLoaderFromRemoteDumpNoStorage() | dump_loader = self.loader | ||||
dump_loader.load(svn_url=self.svn_mirror_url) | dump_loader.load(svn_url=self.svn_mirror_url) | ||||
base_loader = SvnLoaderNoStorage() | self.assertCountContents(19) | ||||
self.assertCountDirectories(17) | |||||
self.assertCountRevisions(6) | |||||
self.assertCountSnapshots(1) | |||||
base_loader = SvnLoaderTest() | |||||
base_loader.load(svn_url=self.svn_mirror_url) | base_loader.load(svn_url=self.svn_mirror_url) | ||||
self.assertEqual(dump_loader.state('content'), | dump_storage_stat = dump_loader.storage.stat_counters() | ||||
base_loader.state('content')) | base_storage_stat = base_loader.storage.stat_counters() | ||||
self.assertEqual(dump_loader.state('directory'), | |||||
base_loader.state('directory')) | self.assertEqual(dump_storage_stat, base_storage_stat) | ||||
self.assertEqual(dump_loader.state('revision'), | |||||
base_loader.state('revision')) | |||||
self.assertEqual(dump_loader.state('snapshot'), | |||||
base_loader.state('snapshot')) | |||||
class SvnLoaderITTest14(BaseSvnLoaderTest): | class SvnLoaderTest14(BaseSvnLoaderTest): | ||||
"""Edge cases: The repository held some user defined svn-properties | """Edge cases: The repository held some user defined svn-properties | ||||
with special encodings, this prevented the repository from | with special encodings, this prevented the repository from | ||||
being loaded even though we do not ingest those information. | being loaded even though we do not ingest those information. | ||||
""" | """ | ||||
def setUp(self): | def setUp(self): | ||||
super().setUp(archive_name='httthttt.tgz', filename='httthttt') | super().setUp(archive_name='httthttt.tgz', filename='httthttt') | ||||
self.loader = SvnLoaderNoStorage() | |||||
def test_load(self): | def test_load(self): | ||||
"""Decoding user defined svn properties error should not fail loading | """Decoding user defined svn properties error should not fail loading | ||||
""" | """ | ||||
# when | # when | ||||
self.loader.load( | self.loader.load( | ||||
svn_url=self.svn_mirror_url, | svn_url=self.svn_mirror_url, | ||||
Show All 9 Lines | def test_load(self): | ||||
'e1e3314e0e9c9d17e6a3f60d6662f48f0e3c2fa3': '7bfb95cef68c1affe8d7f786353213d92abbb2b7', # noqa | 'e1e3314e0e9c9d17e6a3f60d6662f48f0e3c2fa3': '7bfb95cef68c1affe8d7f786353213d92abbb2b7', # noqa | ||||
'1632fd38a8653e9b607c00feb93a41faddfb544c': 'cd6de65c84d9405e7ca45fead02aa10162e30727', # noqa | '1632fd38a8653e9b607c00feb93a41faddfb544c': 'cd6de65c84d9405e7ca45fead02aa10162e30727', # noqa | ||||
'0ad1ebbb92d00721644b0a46d6322d18dbcba848': 'cd6de65c84d9405e7ca45fead02aa10162e30727', # noqa | '0ad1ebbb92d00721644b0a46d6322d18dbcba848': 'cd6de65c84d9405e7ca45fead02aa10162e30727', # noqa | ||||
'94b87c97697d178a9311b018daa5179f7d4ba31e': 'c2128108adecb59a0144339c2e701cd8118cff5a', # noqa | '94b87c97697d178a9311b018daa5179f7d4ba31e': 'c2128108adecb59a0144339c2e701cd8118cff5a', # noqa | ||||
'bd741cf22f0642d88cd0d8b545e8896b898c692d': 'c2128108adecb59a0144339c2e701cd8118cff5a', # noqa | 'bd741cf22f0642d88cd0d8b545e8896b898c692d': 'c2128108adecb59a0144339c2e701cd8118cff5a', # noqa | ||||
last_revision: 'f051d60256b2d89a0ca2704d6f91ad1b0ab44e02', | last_revision: 'f051d60256b2d89a0ca2704d6f91ad1b0ab44e02', | ||||
} | } | ||||
self.assertRevisionsOk(expected_revisions) | self.assertRevisionsContain(expected_revisions) | ||||
expected_snapshot_id = '70487267f682c07e52a2371061369b6cf5bffa47' | expected_snapshot_id = '70487267f682c07e52a2371061369b6cf5bffa47' | ||||
expected_branches = { | expected_branches = { | ||||
'HEAD': { | 'HEAD': { | ||||
'target': last_revision, | 'target': last_revision, | ||||
'target_type': 'revision' | 'target_type': 'revision' | ||||
} | } | ||||
} | } | ||||
self.assertSnapshotOk(expected_snapshot_id, expected_branches) | self.assertSnapshotEqual(expected_snapshot_id, expected_branches) | ||||
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | ||||
self.assertEqual(self.loader.visit_status(), 'full') | self.assertEqual(self.loader.visit_status(), 'full') |
What do you think of: