Page MenuHomeSoftware Heritage
Differential D776 Diff 2450 swh/loader/svn/tests/test_loader.py

Changeset View

Standalone View

View Options

swh/loader/svn/tests/test_loader.py

# Copyright (C) 2016-2018 The Software Heritage developers # Copyright (C) 2016-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import os import os
from unittest import TestCase from unittest import TestCase
from swh.loader.core.tests import BaseLoaderTest, LoaderNoStorage from swh.loader.core.tests import BaseLoaderTest
from swh.loader.svn.loader import (DEFAULT_BRANCH, SvnLoader, from swh.loader.svn.loader import (DEFAULT_BRANCH, SvnLoader,
SvnLoaderFromRemoteDump, build_swh_snapshot) SvnLoaderFromRemoteDump, build_swh_snapshot)
from swh.model import hashutil from swh.model import hashutil
class BaseSvnLoaderTest(BaseLoaderTest):
"""Base test loader class.
In its setup, it's uncompressing a local svn mirror to /tmp.
"""
def setUp(self, archive_name='pkg-gourmet.tgz', filename='pkg-gourmet'):
super().setUp(archive_name=archive_name, filename=filename,
prefix_tmp_folder_name='swh.loader.svn.',
start_path=os.path.dirname(__file__))
self.svn_mirror_url = self.repo_url
class TestSnapshot(TestCase): class TestSnapshot(TestCase):
def test_build_swh_snapshot(self): def test_build_swh_snapshot(self):
ardumontUnsubmitted
Not Done
Inline Actions

What do you think of:

if not loader:
    loader = SvnLoaderMemoryStorage()
self.loader = loader
ardumont: What do you think of: ``` if not loader: loader = SvnLoaderMemoryStorage() self.loader =…
vlorentzUnsubmitted
Not Done
Inline Actions
self.loader = loader or SvnLoaderMemoryStorage()

;)

vlorentz: ``` self.loader = loader or SvnLoaderMemoryStorage() ``` ;)
anlambertAuthorUnsubmitted
Done
Inline Actions

indeed, better

anlambert: indeed, better
ardumontUnsubmitted
Not Done
Inline Actions

\m/

ardumont: \m/
actual_snap = build_swh_snapshot('revision-id') actual_snap = build_swh_snapshot('revision-id')
self.assertEqual(actual_snap, { self.assertEqual(actual_snap, {
'id': None, 'id': None,
'branches': { 'branches': {
DEFAULT_BRANCH: { DEFAULT_BRANCH: {
'target': 'revision-id', 'target': 'revision-id',
'target_type': 'revision', 'target_type': 'revision',
} }
} }
}) })
class LoaderWithState: _LOADER_TEST_CONFIG = {
"""Additional state setup (bypassed by some override for test purposes) 'check_revision': {'limit': 100, 'status': False},
'content_packet_block_size_bytes': 104857600,
'content_packet_size': 10000,
'content_packet_size_bytes': 1073741824,
'content_size_limit': 104857600,
'debug': False,
'directory_packet_size': 2500,
'log_db': 'dbname=softwareheritage-log',
'occurrence_packet_size': 1000,
'release_packet_size': 1000,
'revision_packet_size': 10,
'save_data': False,
'save_data_path': '',
'send_contents': True,
'send_directories': True,
'send_occurrences': True,
'send_releases': True,
'send_revisions': True,
'send_snapshot': True,
'storage': {'args': {}, 'cls': 'memory'},
'temp_directory': '/tmp'
}
class SvnLoaderTest(SvnLoader):
"""An SVNLoader with no persistence.
Context:
Load a new svn repository using the swh policy (so no update).
""" """
def __init__(self): def __init__(self, last_snp_rev={}):
super().__init__() super().__init__()
self.origin = { self.origin = {
'id': 1, 'id': 1,
'url': '/dev/null', 'url': '/dev/null',
'type': 'svn', 'type': 'svn',
} }
self.visit = { self.visit = {
'origin': 1, 'origin': 1,
'visit': 1, 'visit': 1,
} }
self.last_snp_rev = last_snp_rev
def parse_config_file(self, *args, **kwargs):
return _LOADER_TEST_CONFIG
class SvnLoaderNoStorage(LoaderNoStorage, LoaderWithState, SvnLoader):
"""An SVNLoader with no persistence.
Context:
Load a new svn repository using the swh policy (so no update).
"""
def swh_latest_snapshot_revision(self, origin_id, prev_swh_revision=None):
"""We do not know this repository so no revision.
"""
return {}
class SvnLoaderUpdateNoStorage(LoaderNoStorage, LoaderWithState, SvnLoader):
"""An SVNLoader with no persistence.
Context:
Load a known svn repository using the swh policy.
We can either:
- do nothing since it does not contain any new commit (so no
change)
- either check its history is not altered and update in
consequence by loading the new revision
"""
def swh_latest_snapshot_revision(self, origin_id, prev_swh_revision=None): def swh_latest_snapshot_revision(self, origin_id, prev_swh_revision=None):
ardumontUnsubmitted
Not Done
Inline Actions

Do we still do it that way?

I mean, IIRC, it was there to override the svn loader to bypass the real connection with the collaborator storage (no in-memory at the time).

Now that we are using the memory storage, we supposedly can set the collaborator memory storage up to do its job (here to not have any snapshot to return for the specified origin in that particular case).

But that can also be done later ;)

Note: I won't repeat this everywhere but that remark/question stands for all different loader implementations below ;)

ardumont: Do we still do it that way? I mean, IIRC, it was there to override the svn loader to bypass…
vlorentzUnsubmitted
Not Done
Inline Actions

Agreed, all classes with "Storage" in their name should be dropped

vlorentz: Agreed, all classes with "Storage" in their name should be dropped
anlambertAuthorUnsubmitted
Done
Inline Actions

Will see what I can do

anlambert: Will see what I can do
"""Avoid the storage persistence call and return the expected previous """Avoid the storage persistence call and return the expected previous
revision for that repository. revision for that repository.
Check the following for explanation about the hashes: Check the following for explanation about the hashes:
- test_loader.org for (swh policy). - test_loader.org for (swh policy).
- cf. SvnLoaderITTest - cf. SvnLoaderTest
""" """
return { return self.last_snp_rev
'snapshot': 'something', # need a snapshot of sort
'revision': {
'id': hashutil.hash_to_bytes(
'4876cb10aec6f708f7466dddf547567b65f6c39c'),
'parents': [hashutil.hash_to_bytes(
'a3a577948fdbda9d1061913b77a1588695eadb41')],
'directory': hashutil.hash_to_bytes(
'0deab3023ac59398ae467fc4bff5583008af1ee2'),
'target_type': 'revision',
'metadata': {
'extra_headers': [
['svn_repo_uuid',
'3187e211-bb14-4c82-9596-0b59d67cd7f4'],
['svn_revision', '6']
]
}
}
}
class SvnLoaderUpdateHistoryAlteredNoStorage(LoaderNoStorage, LoaderWithState, class BaseSvnLoaderTest(BaseLoaderTest):
SvnLoader): """Base test loader class.
"""Context: Load a known svn repository using the swh policy with its
history altered so we do not update it.
"""
def swh_latest_snapshot_revision(self, origin_id, prev_swh_revision=None):
"""Avoid the storage persistence call and return the expected previous
revision for that repository.
Check the following for explanation about the hashes: In its setup, it's uncompressing a local svn mirror to /tmp.
- test_loader.org for (swh policy).
- cf. SvnLoaderITTest
""" """
return { def setUp(self, archive_name='pkg-gourmet.tgz', filename='pkg-gourmet',
'snapshot': None, loader=None):
'revision': { super().setUp(archive_name=archive_name, filename=filename,
# Changed the revision id's hash to simulate history altered prefix_tmp_folder_name='swh.loader.svn.',
'id': hashutil.hash_to_bytes( start_path=os.path.dirname(__file__))
'badbadbadbadf708f7466dddf547567b65f6c39d'), self.svn_mirror_url = self.repo_url
'parents': [hashutil.hash_to_bytes( self.loader = loader or SvnLoaderTest()
'a3a577948fdbda9d1061913b77a1588695eadb41')], self.storage = self.loader.storage
'directory': hashutil.hash_to_bytes(
'0deab3023ac59398ae467fc4bff5583008af1ee2'),
'target_type': 'revision',
'metadata': {
'extra_headers': [
['svn_repo_uuid',
'3187e211-bb14-4c82-9596-0b59d67cd7f4'],
['svn_revision', b'6']
]
}
}
}
class SvnLoaderITest1(BaseSvnLoaderTest): class SvnLoaderTest1(BaseSvnLoaderTest):
"""Load an unknown svn repository results in new data. """Load an unknown svn repository results in new data.
""" """
def setUp(self):
super().setUp()
self.loader = SvnLoaderNoStorage()
def test_load(self): def test_load(self):
"""Load a new repository results in new swh object and snapshot """Load a new repository results in new swh object and snapshot
""" """
# when # when
self.loader.load( self.loader.load(
svn_url=self.svn_mirror_url, svn_url=self.svn_mirror_url,
destination_path=self.destination_path) destination_path=self.destination_path)
Show All 10 Lines def test_load(self):
'0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71': '669a71cce6c424a81ba42b7dc5d560d32252f0ca', # noqa '0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71': '669a71cce6c424a81ba42b7dc5d560d32252f0ca', # noqa
'95edacc8848369d6fb1608e887d6d2474fd5224f': '008ac97a1118560797c50e3392fa1443acdaa349', # noqa '95edacc8848369d6fb1608e887d6d2474fd5224f': '008ac97a1118560797c50e3392fa1443acdaa349', # noqa
'fef26ea45a520071711ba2b9d16a2985ee837021': '3780effbe846a26751a95a8c95c511fb72be15b4', # noqa 'fef26ea45a520071711ba2b9d16a2985ee837021': '3780effbe846a26751a95a8c95c511fb72be15b4', # noqa
'3f51abf3b3d466571be0855dfa67e094f9ceff1b': 'ffcca9b09c5827a6b8137322d4339c8055c3ee1e', # noqa '3f51abf3b3d466571be0855dfa67e094f9ceff1b': 'ffcca9b09c5827a6b8137322d4339c8055c3ee1e', # noqa
'a3a577948fdbda9d1061913b77a1588695eadb41': '7dc52cc04c3b8bd7c085900d60c159f7b846f866', # noqa 'a3a577948fdbda9d1061913b77a1588695eadb41': '7dc52cc04c3b8bd7c085900d60c159f7b846f866', # noqa
last_revision: '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa last_revision: '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa
} }
self.assertRevisionsOk(expected_revisions) self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1) self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state # FIXME: Check the snapshot's state
# self.assertEqual(self.loader.all_snapshots[0], {}) # self.assertEqual(self.loader.all_snapshots[0], {})
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full') self.assertEqual(self.loader.visit_status(), 'full')
class SvnLoaderITest2(BaseSvnLoaderTest): _last_snp_rev = {
vlorentzUnsubmitted
Not Done
Inline Actions

rename in caps lock (that's a constant). Also probably split snapshot and revision.

vlorentz: rename in caps lock (that's a constant). Also probably split snapshot and revision.
anlambertAuthorUnsubmitted
Done
Inline Actions

ack

anlambert: ack
'snapshot': {
'id': 'something',
'branches': {}
}, # need a snapshot of sort
'revision': {
'id': hashutil.hash_to_bytes(
'4876cb10aec6f708f7466dddf547567b65f6c39c'),
'parents': [hashutil.hash_to_bytes(
'a3a577948fdbda9d1061913b77a1588695eadb41')],
'directory': hashutil.hash_to_bytes(
'0deab3023ac59398ae467fc4bff5583008af1ee2'),
'target_type': 'revision',
'metadata': {
'extra_headers': [
['svn_repo_uuid',
'3187e211-bb14-4c82-9596-0b59d67cd7f4'],
['svn_revision', '6']
]
}
}
}
class SvnLoaderTest2(BaseSvnLoaderTest):
"""Load a visited repository with no new change results in no data """Load a visited repository with no new change results in no data
change. change.
""" """
def setUp(self): def setUp(self):
super().setUp() super().setUp(loader=SvnLoaderTest(last_snp_rev=_last_snp_rev))
vlorentzUnsubmitted
Not Done
Inline Actions

Replace with:

super().setUp(loader=SvnLoaderTest)
self.storage.revision_add([_last_snp_rev['revision']])
self.storage.snapshot_add([_last_snp_rev['snapshot']])

And remove/rename all occurences of last_snp_rev.

vlorentz: Replace with: ``` super().setUp(loader=SvnLoaderTest) self.storage.revision_add([_last_snp_rev…
anlambertAuthorUnsubmitted
Done
Inline Actions

Unfortunately, that's not so simple. I have just tested to remove the swh_latest_snapshot_revision override in the test loader class in favor of only relying to the in-memory storage but numerous errors appeared. So I will keep it this way for the moment.

anlambert: Unfortunately, that's not so simple. I have just tested to remove the…
self.loader = SvnLoaderUpdateNoStorage()
def test_load(self): def test_load(self):
"""Load a repository without new changes results in same snapshot """Load a repository without new changes results in same snapshot
""" """
# when # when
self.loader.load( self.loader.load(
svn_url=self.svn_mirror_url, svn_url=self.svn_mirror_url,
destination_path=self.destination_path) destination_path=self.destination_path)
# then # then
self.assertCountContents(0) self.assertCountContents(0)
self.assertCountDirectories(0) self.assertCountDirectories(0)
self.assertCountRevisions(0) self.assertCountRevisions(0)
self.assertCountReleases(0) self.assertCountReleases(0)
self.assertCountSnapshots(1) self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state # FIXME: Check the snapshot's state
# self.assertEqual(self.loader.all_snapshots[0], {}) # self.assertEqual(self.loader.all_snapshots[0], {})
self.assertEqual(self.loader.load_status(), {'status': 'uneventful'}) self.assertEqual(self.loader.load_status(), {'status': 'uneventful'})
self.assertEqual(self.loader.visit_status(), 'full') self.assertEqual(self.loader.visit_status(), 'full')
class SvnLoaderITest3(BaseSvnLoaderTest): class SvnLoaderTest3(BaseSvnLoaderTest):
"""In this scenario, the dump has been tampered with to modify the """In this scenario, the dump has been tampered with to modify the
commit log. This results in a hash divergence which is commit log. This results in a hash divergence which is
detected at startup. detected at startup.
In effect, that stops the loading and do nothing. In effect, that stops the loading and do nothing.
""" """
def setUp(self): def setUp(self):
last_snp_rev = {
'snapshot': None,
'revision': {
# Changed the revision id's hash to simulate history altered
'id': hashutil.hash_to_bytes(
'badbadbadbadf708f7466dddf547567b65f6c39d'),
'parents': [hashutil.hash_to_bytes(
'a3a577948fdbda9d1061913b77a1588695eadb41')],
'directory': hashutil.hash_to_bytes(
'0deab3023ac59398ae467fc4bff5583008af1ee2'),
'target_type': 'revision',
'metadata': {
'extra_headers': [
['svn_repo_uuid',
'3187e211-bb14-4c82-9596-0b59d67cd7f4'],
['svn_revision', b'6']
]
}
}
}
vlorentzUnsubmitted
Not Done
Inline Actions

kind of a duplicate of the global one.

vlorentz: kind of a duplicate of the global one.
anlambertAuthorUnsubmitted
Done
Inline Actions

ack

anlambert: ack
# the svn repository pkg-gourmet has been updated with changes # the svn repository pkg-gourmet has been updated with changes
super().setUp(archive_name='pkg-gourmet-with-updates.tgz') super().setUp(archive_name='pkg-gourmet-with-updates.tgz',
self.loader = SvnLoaderUpdateHistoryAlteredNoStorage() loader=SvnLoaderTest(last_snp_rev=last_snp_rev))
def test_load(self): def test_load(self):
"""Load known repository with history altered should do nothing """Load known repository with history altered should do nothing
""" """
# when # when
self.loader.load(svn_url=self.svn_mirror_url, self.loader.load(svn_url=self.svn_mirror_url,
destination_path=self.destination_path) destination_path=self.destination_path)
# then # then
# we got the previous run's last revision (rev 6) # we got the previous run's last revision (rev 6)
# so 2 news + 1 old # so 2 news + 1 old
self.assertCountContents(0) self.assertCountContents(0)
self.assertCountDirectories(0) self.assertCountDirectories(0)
self.assertCountRevisions(0) self.assertCountRevisions(0)
self.assertCountReleases(0) self.assertCountReleases(0)
self.assertCountSnapshots(0) self.assertCountSnapshots(0)
# FIXME: Check the snapshot's state # FIXME: Check the snapshot's state
# self.assertEqual(self.loader.all_snapshots[0], {}) # self.assertEqual(self.loader.all_snapshots[0], {})
self.assertEqual(self.loader.load_status(), {'status': 'uneventful'}) self.assertEqual(self.loader.load_status(), {'status': 'uneventful'})
self.assertEqual(self.loader.visit_status(), 'partial') self.assertEqual(self.loader.visit_status(), 'partial')
class SvnLoaderITest4(BaseSvnLoaderTest): class SvnLoaderTest4(BaseSvnLoaderTest):
"""In this scenario, the repository has been updated with new changes. """In this scenario, the repository has been updated with new changes.
The loading visit should result in new objects stored and 1 new The loading visit should result in new objects stored and 1 new
snapshot. snapshot.
""" """
def setUp(self): def setUp(self):
# the svn repository pkg-gourmet has been updated with changes # the svn repository pkg-gourmet has been updated with changes
super().setUp(archive_name='pkg-gourmet-with-updates.tgz') super().setUp(archive_name='pkg-gourmet-with-updates.tgz',
self.loader = SvnLoaderUpdateNoStorage() loader=SvnLoaderTest(last_snp_rev=_last_snp_rev))
def test_process_repository(self): def test_process_repository(self):
"""Process updated repository should yield new objects """Process updated repository should yield new objects
""" """
# when # when
self.loader.load(svn_url=self.svn_mirror_url, self.loader.load(svn_url=self.svn_mirror_url,
destination_path=self.destination_path) destination_path=self.destination_path)
Show All 11 Lines def test_process_repository(self):
# revision hash | directory hash # revision hash | directory hash
'7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa
'38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa
'99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa
'902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa
last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa
} }
self.assertRevisionsOk(expected_revisions) self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1) self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state # FIXME: Check the snapshot's state
# self.assertEqual(self.loader.all_snapshots[0], {}) # self.assertEqual(self.loader.all_snapshots[0], {})
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full') self.assertEqual(self.loader.visit_status(), 'full')
class SvnLoaderITTest5(BaseSvnLoaderTest): class SvnLoaderTest5(BaseSvnLoaderTest):
"""Context: """Context:
- Repository already injected with successful data - Repository already injected with successful data
- New visit from scratch done with successful load - New visit from scratch done with successful load
""" """
def setUp(self): def setUp(self):
# the svn repository pkg-gourmet has been updated with changes # the svn repository pkg-gourmet has been updated with changes
super().setUp(archive_name='pkg-gourmet-with-updates.tgz') super().setUp(archive_name='pkg-gourmet-with-updates.tgz',
self.loader = SvnLoaderUpdateNoStorage() loader=SvnLoaderTest(last_snp_rev=_last_snp_rev))
def test_load(self): def test_load(self):
"""Load an existing repository from scratch yields same swh objects """Load an existing repository from scratch yields same swh objects
""" """
# when # when
self.loader.load(svn_url=self.svn_mirror_url, self.loader.load(svn_url=self.svn_mirror_url,
destination_path=self.destination_path, destination_path=self.destination_path,
Show All 15 Lines def test_load(self):
'4876cb10aec6f708f7466dddf547567b65f6c39c': '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa '4876cb10aec6f708f7466dddf547567b65f6c39c': '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa
'7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa
'38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa
'99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa
'902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa
'171dc35522bfd17dda4e90a542a0377fb2fc707a': 'fd24a76c87a3207428e06612b49860fc78e9f6dc', # noqa '171dc35522bfd17dda4e90a542a0377fb2fc707a': 'fd24a76c87a3207428e06612b49860fc78e9f6dc', # noqa
} }
self.assertRevisionsOk(expected_revisions) self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1) self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state # FIXME: Check the snapshot's state
# self.assertEqual(self.loader.all_snapshots[0], {}) # self.assertEqual(self.loader.all_snapshots[0], {})
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full') self.assertEqual(self.loader.visit_status(), 'full')
class SvnLoaderWithPreviousRevisionNoStorage(LoaderNoStorage, LoaderWithState, class SvnLoaderTest6(BaseSvnLoaderTest):
SvnLoader): """Context:
"""An SVNLoader with no persistence. - repository already visited with load successful
- Changes on existing repository
Context: Load a known svn repository using the swh policy with its - New Visit done with successful new data
history altered so we do not update it.
"""
def swh_latest_snapshot_revision(self, origin_id, prev_swh_revision=None):
"""Avoid the storage persistence call and return the expected previous
revision for that repository.
Check the following for explanation about the hashes:
- test_loader.org for (swh policy).
- cf. SvnLoaderITTest
""" """
return { def setUp(self):
last_snp_rev = {
'snapshot': None, 'snapshot': None,
'revision': { 'revision': {
'id': hashutil.hash_to_bytes( 'id': hashutil.hash_to_bytes(
'4876cb10aec6f708f7466dddf547567b65f6c39c'), '4876cb10aec6f708f7466dddf547567b65f6c39c'),
'parents': [hashutil.hash_to_bytes( 'parents': [hashutil.hash_to_bytes(
'a3a577948fdbda9d1061913b77a1588695eadb41')], 'a3a577948fdbda9d1061913b77a1588695eadb41')],
'directory': hashutil.hash_to_bytes( 'directory': hashutil.hash_to_bytes(
'0deab3023ac59398ae467fc4bff5583008af1ee2'), '0deab3023ac59398ae467fc4bff5583008af1ee2'),
'target_type': 'revision', 'target_type': 'revision',
'metadata': { 'metadata': {
'extra_headers': [ 'extra_headers': [
['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], # noqa ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], # noqa
['svn_revision', '6'] ['svn_revision', '6']
] ]
} }
} }
} }
super().setUp(archive_name='pkg-gourmet-with-updates.tgz',
loader=SvnLoaderTest(last_snp_rev=last_snp_rev))
class SvnLoaderITTest6(BaseSvnLoaderTest):
"""Context:
- repository already visited with load successful
- Changes on existing repository
- New Visit done with successful new data
"""
def setUp(self):
super().setUp(archive_name='pkg-gourmet-with-updates.tgz')
self.loader = SvnLoaderWithPreviousRevisionNoStorage()
def test_load(self): def test_load(self):
"""Load from partial previous visit result in new changes """Load from partial previous visit result in new changes
""" """
# when # when
self.loader.load( self.loader.load(
svn_url=self.svn_mirror_url, svn_url=self.svn_mirror_url,
Show All 12 Lines def test_load(self):
# revision hash | directory hash # revision hash | directory hash
'7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa
'38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa
'99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa
'902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa
last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa
} }
self.assertRevisionsOk(expected_revisions) self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1) self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state # FIXME: Check the snapshot's state
# self.assertEqual(self.loader.all_snapshots[0], {}) # self.assertEqual(self.loader.all_snapshots[0], {})
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full') self.assertEqual(self.loader.visit_status(), 'full')
class SvnLoaderITest7(BaseSvnLoaderTest): class SvnLoaderTest7(BaseSvnLoaderTest):
"""Context: """Context:
- repository already visited with load successful - repository already visited with load successful
- Changes on existing repository - Changes on existing repository
- New Visit done with successful new data - New Visit done with successful new data
""" """
def setUp(self): def setUp(self):
super().setUp(archive_name='pkg-gourmet-with-updates.tgz') super().setUp(archive_name='pkg-gourmet-with-updates.tgz',
self.loader = SvnLoaderUpdateNoStorage() loader=SvnLoaderTest(last_snp_rev=_last_snp_rev))
def test_load(self): def test_load(self):
"""Load known and partial repository should start from last visit """Load known and partial repository should start from last visit
""" """
previous_unfinished_revision = { previous_unfinished_revision = {
'id': hashutil.hash_to_bytes( 'id': hashutil.hash_to_bytes(
'a3a577948fdbda9d1061913b77a1588695eadb41'), 'a3a577948fdbda9d1061913b77a1588695eadb41'),
Show All 29 Lines def test_load(self):
# revision hash | directory hash # revision hash | directory hash
'7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa
'38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa
'99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa
'902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa
last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa
} }
self.assertRevisionsOk(expected_revisions) self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1) self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state # FIXME: Check the snapshot's state
# self.assertEqual(self.loader.all_snapshots[0], {}) # self.assertEqual(self.loader.all_snapshots[0], {})
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full') self.assertEqual(self.loader.visit_status(), 'full')
class SvnLoaderUpdateLessRecentNoStorage(LoaderNoStorage, LoaderWithState, class SvnLoaderTest8(BaseSvnLoaderTest):
SvnLoader):
"""Context: """Context:
Load a known svn repository. The last visit seen is less
recent than a previous unfinished crawl.
""" - Previous visit on existing repository done
def swh_latest_snapshot_revision(self, origin_id, prev_swh_revision=None): - Starting the loading from the last unfinished visit
"""Avoid the storage persistence call and return the expected previous - New objects are created (1 new snapshot)
revision for that repository.
Check the following for explanation about the hashes:
- test_loader.org for (swh policy).
- cf. SvnLoaderITTest
""" """
return { def setUp(self):
last_snp_rev = {
'snapshot': None, 'snapshot': None,
'revision': { 'revision': {
'id': hashutil.hash_to_bytes( 'id': hashutil.hash_to_bytes(
'a3a577948fdbda9d1061913b77a1588695eadb41'), 'a3a577948fdbda9d1061913b77a1588695eadb41'),
'parents': [hashutil.hash_to_bytes( 'parents': [hashutil.hash_to_bytes(
'3f51abf3b3d466571be0855dfa67e094f9ceff1b')], '3f51abf3b3d466571be0855dfa67e094f9ceff1b')],
'directory': hashutil.hash_to_bytes( 'directory': hashutil.hash_to_bytes(
'7dc52cc04c3b8bd7c085900d60c159f7b846f866'), '7dc52cc04c3b8bd7c085900d60c159f7b846f866'),
'target_type': 'revision', 'target_type': 'revision',
'metadata': { 'metadata': {
'extra_headers': [ 'extra_headers': [
['svn_repo_uuid', ['svn_repo_uuid',
'3187e211-bb14-4c82-9596-0b59d67cd7f4'], '3187e211-bb14-4c82-9596-0b59d67cd7f4'],
['svn_revision', '5'] ['svn_revision', '5']
] ]
} }
} }
} }
super().setUp(archive_name='pkg-gourmet-with-updates.tgz',
loader=SvnLoaderTest(last_snp_rev=last_snp_rev))
class SvnLoaderITest8(BaseSvnLoaderTest):
"""Context:
- Previous visit on existing repository done
- Starting the loading from the last unfinished visit
- New objects are created (1 new snapshot)
"""
def setUp(self):
super().setUp(archive_name='pkg-gourmet-with-updates.tgz')
self.loader = SvnLoaderUpdateLessRecentNoStorage()
def test_load(self): def test_load(self):
"""Load repository should yield revisions starting from last visit """Load repository should yield revisions starting from last visit
""" """
previous_unfinished_revision = { previous_unfinished_revision = {
'id': hashutil.hash_to_bytes( 'id': hashutil.hash_to_bytes(
'4876cb10aec6f708f7466dddf547567b65f6c39c'), '4876cb10aec6f708f7466dddf547567b65f6c39c'),
Show All 28 Lines def test_load(self):
# revision hash | directory hash # revision hash | directory hash
'7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa
'38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa
'99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa
'902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa
last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa
} }
self.assertRevisionsOk(expected_revisions) self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1) self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state # FIXME: Check the snapshot's state
# self.assertEqual(self.loader.all_snapshots[0], {}) # self.assertEqual(self.loader.all_snapshots[0], {})
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full') self.assertEqual(self.loader.visit_status(), 'full')
class SvnLoaderTTest9(BaseSvnLoaderTest): class SvnLoaderTTest9(BaseSvnLoaderTest):
"""Check that a svn repo containing a versioned file with CRLF line """Check that a svn repo containing a versioned file with CRLF line
endings with svn:eol-style property set to 'native' (this is a endings with svn:eol-style property set to 'native' (this is a
violation of svn specification as the file should have been violation of svn specification as the file should have been
stored with LF line endings) can be loaded anyway. stored with LF line endings) can be loaded anyway.
""" """
def setUp(self): def setUp(self):
super().setUp(archive_name='mediawiki-repo-r407-eol-native-crlf.tgz', super().setUp(archive_name='mediawiki-repo-r407-eol-native-crlf.tgz',
filename='mediawiki-repo-r407-eol-native-crlf') filename='mediawiki-repo-r407-eol-native-crlf')
self.loader = SvnLoaderNoStorage()
def test_process_repository(self): def test_process_repository(self):
"""Load repository with CRLF endings (svn:eol-style: native) is ok """Load repository with CRLF endings (svn:eol-style: native) is ok
""" # noqa """ # noqa
# when # when
self.loader.load(svn_url=self.svn_mirror_url, self.loader.load(svn_url=self.svn_mirror_url,
destination_path=self.destination_path) destination_path=self.destination_path)
expected_revisions = { expected_revisions = {
'7da4975c363101b819756d33459f30a866d01b1b': 'f63637223ee0f7d4951ffd2d4d9547a4882c5d8b' # noqa '7da4975c363101b819756d33459f30a866d01b1b': 'f63637223ee0f7d4951ffd2d4d9547a4882c5d8b' # noqa
} }
self.assertRevisionsOk(expected_revisions) self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1) self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state # FIXME: Check the snapshot's state
# self.assertEqual(self.loader.all_snapshots[0], {}) # self.assertEqual(self.loader.all_snapshots[0], {})
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full') self.assertEqual(self.loader.visit_status(), 'full')
class SvnLoaderITest10(BaseSvnLoaderTest): # noqa class SvnLoaderTest10(BaseSvnLoaderTest): # noqa
"""Check that a svn repo containing a versioned file with mixed """Check that a svn repo containing a versioned file with mixed
CRLF/LF line endings with svn:eol-style property set to 'native' CRLF/LF line endings with svn:eol-style property set to 'native'
(this is a violation of svn specification as mixed line endings (this is a violation of svn specification as mixed line endings
for textual content should not be stored when the svn:eol-style for textual content should not be stored when the svn:eol-style
property is set) can be loaded anyway. property is set) can be loaded anyway.
""" """
def setUp(self): def setUp(self):
super().setUp( super().setUp(
archive_name='pyang-repo-r343-eol-native-mixed-lf-crlf.tgz', archive_name='pyang-repo-r343-eol-native-mixed-lf-crlf.tgz',
filename='pyang-repo-r343-eol-native-mixed-lf-crlf') filename='pyang-repo-r343-eol-native-mixed-lf-crlf')
self.loader = SvnLoaderNoStorage()
def test_load(self): def test_load(self):
"""Load repo with mixed CRLF/LF endings (svn:eol-style:native) is ok """Load repo with mixed CRLF/LF endings (svn:eol-style:native) is ok
""" """
self.loader.load(svn_url=self.svn_mirror_url, self.loader.load(svn_url=self.svn_mirror_url,
destination_path=self.destination_path) destination_path=self.destination_path)
expected_revisions = { expected_revisions = {
'9c6962eeb9164a636c374be700672355e34a98a7': '16aa6b6271f3456d4643999d234cf39fe3d0cc5a' # noqa '9c6962eeb9164a636c374be700672355e34a98a7': '16aa6b6271f3456d4643999d234cf39fe3d0cc5a' # noqa
} }
self.assertRevisionsOk(expected_revisions) self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1) self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state # FIXME: Check the snapshot's state
# self.assertEqual(self.loader.all_snapshots[0], {}) # self.assertEqual(self.loader.all_snapshots[0], {})
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full') self.assertEqual(self.loader.visit_status(), 'full')
class SvnLoaderITest11(BaseSvnLoaderTest): class SvnLoaderTest11(BaseSvnLoaderTest):
"""Context: """Context:
- Repository with svn:external (which is not deal with for now) - Repository with svn:external (which is not deal with for now)
- Visit is partial with as much data loaded as possible - Visit is partial with as much data loaded as possible
""" """
def setUp(self): def setUp(self):
super().setUp(archive_name='pkg-gourmet-with-external-id.tgz') super().setUp(archive_name='pkg-gourmet-with-external-id.tgz')
self.loader = SvnLoaderNoStorage()
def test_load(self): def test_load(self):
"""Repository with svn:externals property, will stop raising an error """Repository with svn:externals property, will stop raising an error
""" """
previous_unfinished_revision = None previous_unfinished_revision = None
# when # when
Show All 29 Lines def test_load(self):
'ded78810401fd354ffe894aa4a1e5c7d30a645d1': 'b0a648b02e55a4dce356ac35187a058f89694ec7', # noqa 'ded78810401fd354ffe894aa4a1e5c7d30a645d1': 'b0a648b02e55a4dce356ac35187a058f89694ec7', # noqa
'4ee95e39358712f53c4fc720da3fafee9249ed19': 'c3c98df624733fef4e592bef983f93e2ed02b179', # noqa '4ee95e39358712f53c4fc720da3fafee9249ed19': 'c3c98df624733fef4e592bef983f93e2ed02b179', # noqa
'ffa901b69ca0f46a2261f42948838d19709cb9f8': 'c3c98df624733fef4e592bef983f93e2ed02b179', # noqa 'ffa901b69ca0f46a2261f42948838d19709cb9f8': 'c3c98df624733fef4e592bef983f93e2ed02b179', # noqa
'0148ae3eaa520b73a50802c59f3f416b7a36cf8c': '844d4646d6c2b4f3a3b2b22ab0ee38c7df07bab2', # noqa '0148ae3eaa520b73a50802c59f3f416b7a36cf8c': '844d4646d6c2b4f3a3b2b22ab0ee38c7df07bab2', # noqa
last_revision: '0de6e75d2b79ec90d00a3a7611aa3861b2e4aa5e', # noqa last_revision: '0de6e75d2b79ec90d00a3a7611aa3861b2e4aa5e', # noqa
} }
# The last revision being the one used later to start back from # The last revision being the one used later to start back from
self.assertRevisionsOk(expected_revisions) self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1) self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state # FIXME: Check the snapshot's state
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'partial') self.assertEqual(self.loader.visit_status(), 'partial')
class SvnLoaderITest12(BaseSvnLoaderTest): class SvnLoaderTest12(BaseSvnLoaderTest):
"""Edge cases: """Edge cases:
- first create a file and commit it. - first create a file and commit it.
Remove it, then add folder holding the same name, commit. Remove it, then add folder holding the same name, commit.
- do the same scenario with symbolic link (instead of file) - do the same scenario with symbolic link (instead of file)
""" """
def setUp(self): def setUp(self):
super().setUp( super().setUp(
archive_name='pkg-gourmet-with-edge-case-links-and-files.tgz') archive_name='pkg-gourmet-with-edge-case-links-and-files.tgz')
self.loader = SvnLoaderNoStorage()
def test_load(self): def test_load(self):
"""File/Link removed prior to folder with same name creation is ok """File/Link removed prior to folder with same name creation is ok
""" """
previous_unfinished_revision = None previous_unfinished_revision = None
# when # when
Show All 25 Lines def test_load(self):
'bb78300cc1ac9119eb6fffa9e9fa04a7f9340b11': 'ee995a0d85f6917c75bcee3aa448bea7726b265d', # noqa 'bb78300cc1ac9119eb6fffa9e9fa04a7f9340b11': 'ee995a0d85f6917c75bcee3aa448bea7726b265d', # noqa
'f2e01111329f84580dc3febb1fd45515692c5886': 'e2baec7b6a5543758e9c73695bc847db0a4f7941', # noqa 'f2e01111329f84580dc3febb1fd45515692c5886': 'e2baec7b6a5543758e9c73695bc847db0a4f7941', # noqa
'1a0f70c34e211f073e1be3435ecf6f0dd7700267': 'e7536e721fa806c19971b749c091c144b2f2b88e', # noqa '1a0f70c34e211f073e1be3435ecf6f0dd7700267': 'e7536e721fa806c19971b749c091c144b2f2b88e', # noqa
'0c612a23d293cc3100496a54ae4ad13d750efe4c': '2123d12749294bbfb54e73f9d73fac658aabb266', # noqa '0c612a23d293cc3100496a54ae4ad13d750efe4c': '2123d12749294bbfb54e73f9d73fac658aabb266', # noqa
'69a53d972e2f863acbbbda546d9da96287af6a88': '13896cb96ec004140ce5be8852fee8c29830d9c7', # noqa '69a53d972e2f863acbbbda546d9da96287af6a88': '13896cb96ec004140ce5be8852fee8c29830d9c7', # noqa
last_revision: '6b1e0243768ff9ac060064b2eeade77e764ffc82', # noqa last_revision: '6b1e0243768ff9ac060064b2eeade77e764ffc82', # noqa
} }
self.assertRevisionsOk(expected_revisions) self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1) self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state # FIXME: Check the snapshot's state
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full') self.assertEqual(self.loader.visit_status(), 'full')
class SvnLoaderITTest13(BaseSvnLoaderTest): class SvnLoaderTest13(BaseSvnLoaderTest):
"""Edge cases: """Edge cases:
- wrong symbolic link - wrong symbolic link
- wrong symbolic link with empty space names - wrong symbolic link with empty space names
""" """
def setUp(self): def setUp(self):
super().setUp( super().setUp(
archive_name='pkg-gourmet-with-wrong-link-cases.tgz') archive_name='pkg-gourmet-with-wrong-link-cases.tgz')
self.loader = SvnLoaderNoStorage()
def test_load(self): def test_load(self):
"""Wrong link or empty space-named link should be ok """Wrong link or empty space-named link should be ok
""" """
# when # when
self.loader.load( self.loader.load(
svn_url=self.svn_mirror_url, svn_url=self.svn_mirror_url,
Show All 25 Lines def test_load(self):
'1a0f70c34e211f073e1be3435ecf6f0dd7700267': 'e7536e721fa806c19971b749c091c144b2f2b88e', # noqa '1a0f70c34e211f073e1be3435ecf6f0dd7700267': 'e7536e721fa806c19971b749c091c144b2f2b88e', # noqa
'0c612a23d293cc3100496a54ae4ad13d750efe4c': '2123d12749294bbfb54e73f9d73fac658aabb266', # noqa '0c612a23d293cc3100496a54ae4ad13d750efe4c': '2123d12749294bbfb54e73f9d73fac658aabb266', # noqa
'69a53d972e2f863acbbbda546d9da96287af6a88': '13896cb96ec004140ce5be8852fee8c29830d9c7', # noqa '69a53d972e2f863acbbbda546d9da96287af6a88': '13896cb96ec004140ce5be8852fee8c29830d9c7', # noqa
'3f43af2578fccf18b0d4198e48563da7929dc608': '6b1e0243768ff9ac060064b2eeade77e764ffc82', # noqa '3f43af2578fccf18b0d4198e48563da7929dc608': '6b1e0243768ff9ac060064b2eeade77e764ffc82', # noqa
'4ab5fc264732cd474d2e695c5ac66e4933bdad74': '9a1f5e3961db89422250ce6c1441476f40d65205', # noqa '4ab5fc264732cd474d2e695c5ac66e4933bdad74': '9a1f5e3961db89422250ce6c1441476f40d65205', # noqa
last_revision: 'd853d9628f6f0008d324fed27dadad00ce48bc62', # noqa last_revision: 'd853d9628f6f0008d324fed27dadad00ce48bc62', # noqa
} }
self.assertRevisionsOk(expected_revisions) self.assertRevisionsContain(expected_revisions)
self.assertCountSnapshots(1) self.assertCountSnapshots(1)
# FIXME: Check the snapshot's state # FIXME: Check the snapshot's state
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full') self.assertEqual(self.loader.visit_status(), 'full')
class SvnLoaderFromRemoteDumpNoStorage(LoaderNoStorage, LoaderWithState, class SvnLoaderTestFromRemoteDump(SvnLoaderTest, SvnLoaderFromRemoteDump):
SvnLoaderFromRemoteDump): pass
"""A SvnLoaderFromRemoteDump with no persistence.
Context:
Load a remote svn repository from a generated dump file.
"""
def swh_latest_snapshot_revision(self, origin_id, prev_swh_revision=None):
"""We do not know this repository so no revision.
""" class SvnLoaderFromRemoteDumpTest(BaseSvnLoaderTest):
return {}
class SvnLoaderFromRemoteDump(BaseSvnLoaderTest):
""" """
Check that the results obtained with the remote svn dump loader Check that the results obtained with the remote svn dump loader
and the base svn loader are the same. and the base svn loader are the same.
""" """
def setUp(self): def setUp(self):
super().setUp(archive_name='pkg-gourmet.tgz') super().setUp(archive_name='pkg-gourmet.tgz',
loader=SvnLoaderTestFromRemoteDump())
def test_load(self): def test_load(self):
""" """
Compare results of remote dump loader and base loader Compare results of remote dump loader and base loader
""" """
dump_loader = SvnLoaderFromRemoteDumpNoStorage() dump_loader = self.loader
dump_loader.load(svn_url=self.svn_mirror_url) dump_loader.load(svn_url=self.svn_mirror_url)
base_loader = SvnLoaderNoStorage() self.assertCountContents(19)
self.assertCountDirectories(17)
self.assertCountRevisions(6)
self.assertCountSnapshots(1)
base_loader = SvnLoaderTest()
base_loader.load(svn_url=self.svn_mirror_url) base_loader.load(svn_url=self.svn_mirror_url)
self.assertEqual(dump_loader.state('content'), dump_storage_stat = dump_loader.storage.stat_counters()
base_loader.state('content')) base_storage_stat = base_loader.storage.stat_counters()
self.assertEqual(dump_loader.state('directory'),
base_loader.state('directory')) self.assertEqual(dump_storage_stat, base_storage_stat)
self.assertEqual(dump_loader.state('revision'),
base_loader.state('revision'))
self.assertEqual(dump_loader.state('snapshot'),
base_loader.state('snapshot'))
class SvnLoaderITTest14(BaseSvnLoaderTest): class SvnLoaderTest14(BaseSvnLoaderTest):
"""Edge cases: The repository held some user defined svn-properties """Edge cases: The repository held some user defined svn-properties
with special encodings, this prevented the repository from with special encodings, this prevented the repository from
being loaded even though we do not ingest those information. being loaded even though we do not ingest those information.
""" """
def setUp(self): def setUp(self):
super().setUp(archive_name='httthttt.tgz', filename='httthttt') super().setUp(archive_name='httthttt.tgz', filename='httthttt')
self.loader = SvnLoaderNoStorage()
def test_load(self): def test_load(self):
"""Decoding user defined svn properties error should not fail loading """Decoding user defined svn properties error should not fail loading
""" """
# when # when
self.loader.load( self.loader.load(
svn_url=self.svn_mirror_url, svn_url=self.svn_mirror_url,
Show All 9 Lines def test_load(self):
'e1e3314e0e9c9d17e6a3f60d6662f48f0e3c2fa3': '7bfb95cef68c1affe8d7f786353213d92abbb2b7', # noqa 'e1e3314e0e9c9d17e6a3f60d6662f48f0e3c2fa3': '7bfb95cef68c1affe8d7f786353213d92abbb2b7', # noqa
'1632fd38a8653e9b607c00feb93a41faddfb544c': 'cd6de65c84d9405e7ca45fead02aa10162e30727', # noqa '1632fd38a8653e9b607c00feb93a41faddfb544c': 'cd6de65c84d9405e7ca45fead02aa10162e30727', # noqa
'0ad1ebbb92d00721644b0a46d6322d18dbcba848': 'cd6de65c84d9405e7ca45fead02aa10162e30727', # noqa '0ad1ebbb92d00721644b0a46d6322d18dbcba848': 'cd6de65c84d9405e7ca45fead02aa10162e30727', # noqa
'94b87c97697d178a9311b018daa5179f7d4ba31e': 'c2128108adecb59a0144339c2e701cd8118cff5a', # noqa '94b87c97697d178a9311b018daa5179f7d4ba31e': 'c2128108adecb59a0144339c2e701cd8118cff5a', # noqa
'bd741cf22f0642d88cd0d8b545e8896b898c692d': 'c2128108adecb59a0144339c2e701cd8118cff5a', # noqa 'bd741cf22f0642d88cd0d8b545e8896b898c692d': 'c2128108adecb59a0144339c2e701cd8118cff5a', # noqa
last_revision: 'f051d60256b2d89a0ca2704d6f91ad1b0ab44e02', last_revision: 'f051d60256b2d89a0ca2704d6f91ad1b0ab44e02',
} }
self.assertRevisionsOk(expected_revisions) self.assertRevisionsContain(expected_revisions)
expected_snapshot_id = '70487267f682c07e52a2371061369b6cf5bffa47' expected_snapshot_id = '70487267f682c07e52a2371061369b6cf5bffa47'
expected_branches = { expected_branches = {
'HEAD': { 'HEAD': {
'target': last_revision, 'target': last_revision,
'target_type': 'revision' 'target_type': 'revision'
} }
} }
self.assertSnapshotOk(expected_snapshot_id, expected_branches) self.assertSnapshotEqual(expected_snapshot_id, expected_branches)
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full') self.assertEqual(self.loader.visit_status(), 'full')
About · Developer information · Legal