Page MenuHomeSoftware Heritage
Files F9697396

No OneTemporary
Actions

View Options

diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py
index 1750b2b..d440849 100644
--- a/swh/loader/svn/loader.py
+++ b/swh/loader/svn/loader.py
@@ -1,506 +1,508 @@
# Copyright (C) 2015-2017 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
"""Loader in charge of injecting either new or existing svn mirrors to
swh-storage.
"""
import abc
import os
import shutil
from swh.core import utils
from swh.model import git, hashutil
from swh.model.git import GitType
from swh.loader.core.loader import SWHLoader
from . import svn, converters
from .utils import hashtree, init_svn_repo_from_archive_dump
class SvnLoaderEventful(ValueError):
"""A wrapper exception to transit the swh_revision onto which the
loading failed.
"""
def __init__(self, e, swh_revision):
super().__init__(e)
self.swh_revision = swh_revision
class SvnLoaderUneventful(ValueError):
- pass
+ def __init__(self, e, *args):
+ super().__init__(e, *args)
class SvnLoaderHistoryAltered(ValueError):
- pass
+ def __init__(self, e, *args):
+ super().__init__(e, *args)
class BaseSvnLoader(SWHLoader, metaclass=abc.ABCMeta):
"""Base Svn loader to load one svn repository according to specific
policies (only swh one now).
The main entry point of this is (no need to override it)::
def load(self, origin_visit, last_known_swh_revision=None): pass
Inherit this class and then override the following functions::
def build_swh_revision(self, rev, commit, dir_id, parents):
# This is in charge of converting an svn revision to a compliant
# swh revision
def process_repository(self):
# This is in charge of processing the actual svn repository and
# store the result to swh storage.
"""
CONFIG_BASE_FILENAME = 'loader/svn'
ADDITIONAL_CONFIG = {
'check_revision': ('int', 1000),
}
def __init__(self):
super().__init__(logging_class='swh.loader.svn.SvnLoader')
self.check_revision = self.config['check_revision']
@abc.abstractmethod
def swh_revision_hash_tree_at_svn_revision(self, revision):
"""Compute and return the hash tree at a given svn revision.
Args:
rev (int): the svn revision we want to check
Returns:
The hash tree directory as bytes.
"""
pass
@abc.abstractmethod
def get_svn_repo(self, svn_url, destination_path, origin):
"""Instantiates the needed svnrepo collaborator to permit reading svn
repository.
Args:
svn_url: the svn repository url to read from
destination_path: the local path on disk to compute data
origin: the corresponding origin
Returns:
Instance of :mod:`swh.loader.svn.svn` clients
"""
raise NotImplementedError
@abc.abstractmethod
def build_swh_revision(self, rev, commit, dir_id, parents):
"""Convert an svn revision to an swh one according to the loader's
policy (git-svn or swh).
Args:
rev: the svn revision number
commit: dictionary with keys: author\_name, author\_date, rev,
message
dir_id: the hash tree computation
parents: the revision's parents
Returns:
The swh revision
"""
raise NotImplementedError
@abc.abstractmethod
def process_repository(self, origin_visit, last_known_swh_revision=None):
"""The main idea of this function is to:
- iterate over the svn commit logs
- extract the svn commit log metadata
- compute the hashes from the current directory down to the file
- compute the equivalent swh revision
- send all those objects for storage
- create an swh occurrence pointing to the last swh revision seen
- send that occurrence for storage in swh-storage.
"""
raise NotImplementedError
def process_svn_revisions(self, svnrepo, revision_start, revision_end,
revision_parents):
"""Process revisions from revision_start to revision_end and send to
swh for storage.
At each svn revision, checkout the repository, compute the
tree hash and blobs and send for swh storage to store.
Then computes and yields the swh revision.
Note that at every self.check_revision, an svn export is done
and a hash tree is computed to check that no divergence
occurred.
Yields:
swh revision as a dictionary with keys, sha1_git, sha1, etc...
"""
gen_revs = svnrepo.swh_hash_data_per_revision(
revision_start,
revision_end)
swh_revision = None
count = 0
for rev, nextrev, commit, objects_per_path in gen_revs:
count += 1
# Send the associated contents/directories
self.maybe_load_contents(
git.objects_per_type(GitType.BLOB, objects_per_path))
self.maybe_load_directories(
git.objects_per_type(GitType.TREE, objects_per_path))
# compute the fs tree's checksums
dir_id = objects_per_path[b'']['checksums']['sha1_git']
swh_revision = self.build_swh_revision(
rev, commit, dir_id, revision_parents[rev])
swh_revision['id'] = git.compute_revision_sha1_git(
swh_revision)
self.log.debug('rev: %s, swhrev: %s, dir: %s' % (
rev,
hashutil.hash_to_hex(swh_revision['id']),
hashutil.hash_to_hex(dir_id)))
if (count % self.check_revision) == 0: # hash computation check
self.log.info('Checking hash computations on revision %s...' %
rev)
checked_dir_id = self.swh_revision_hash_tree_at_svn_revision(
rev)
if checked_dir_id != dir_id:
err = 'Hash tree computation divergence detected (%s != %s), stopping!' % ( # noqa
hashutil.hash_to_hex(dir_id),
hashutil.hash_to_hex(checked_dir_id)
)
raise ValueError(err)
if nextrev:
revision_parents[nextrev] = [swh_revision['id']]
yield swh_revision
def process_swh_revisions(self,
svnrepo,
revision_start,
revision_end,
revision_parents):
"""Process and store revision to swh (sent by blocks of
revision_packet_size)
Returns:
The latest revision stored.
"""
try:
swh_revision_gen = self.process_svn_revisions(svnrepo,
revision_start,
revision_end,
revision_parents)
revs = []
for revisions in utils.grouper(
swh_revision_gen,
self.config['revision_packet_size']):
revs = list(revisions)
self.log.info('Processed %s revisions: [%s, ...]' % (
len(revs), hashutil.hash_to_hex(revs[0]['id'])))
self.maybe_load_revisions(revs)
except Exception as e:
if revs:
# flush remaining revisions
self.maybe_load_revisions(revs)
# Take the last one as the last known revisions
known_swh_rev = revs[-1]
# Then notify something is wrong, and we stopped at that rev.
raise SvnLoaderEventful(e, swh_revision={
'id': known_swh_rev['id'],
})
raise e
return revs[-1]
def process_swh_occurrence(self, revision, origin_visit):
"""Process and load the occurrence pointing to the latest revision.
"""
occ = converters.build_swh_occurrence(
revision['id'], origin_visit['origin'], origin_visit['visit'])
self.log.debug('occ: %s' % occ)
self.maybe_load_occurrences([occ])
def prepare(self, *args, **kwargs):
+ self.args = args
+
destination_path = kwargs['destination_path']
# local svn url
svn_url = kwargs['svn_url']
origin_url = kwargs.get('origin_url')
self.visit_date = kwargs.get('visit_date')
origin = {
'url': origin_url if origin_url else svn_url,
'type': 'svn',
}
self.origin_id = self.send_origin(origin)
origin['id'] = self.origin_id
self.origin = origin
if 'swh_revision' in kwargs:
self.last_known_swh_revision = hashutil.hash_to_bytes(
kwargs['swh_revision'])
else:
self.last_known_swh_revision = None
self.svnrepo = self.get_svn_repo(svn_url, destination_path, origin)
self.fetch_history_id = self.open_fetch_history()
def get_origin(self):
"""Retrieve the origin we are working with.
"""
return self.origin # set in prepare method
def fetch_data(self):
"""We need to fetch and stream the data to store directly. So
fetch_data do actually nothing. The method ``store_data`` below is in
charge to do everything, fetch and store.
"""
pass
def store_data(self):
"""We need to fetch and stream the data to store directly because
there is too much data and state changes. Everything is
intertwined together (We receive patch and apply on disk and
compute at the hashes at the same time)
So every data to fetch and store is done here.
Note:
origin_visit and last_known_swh_revision must have been set in the
prepare method.
"""
origin_visit = {'origin': self.origin_id, 'visit': self.visit}
try:
latest_rev = self.process_repository(origin_visit,
self.last_known_swh_revision)
except SvnLoaderEventful as e:
self.log.error('Eventful partial visit. Detail: %s' % e)
latest_rev = e.swh_revision
self.process_swh_occurrence(latest_rev, origin_visit)
raise
except (SvnLoaderHistoryAltered, SvnLoaderUneventful) as e:
self.log.error('Uneventful visit. Detail: %s' % e)
raise
except Exception as e:
raise
else:
self.process_swh_occurrence(latest_rev, origin_visit)
class SWHSvnLoader(BaseSvnLoader):
"""Swh svn loader is the main implementation destined for production.
This implementation is able to deal with update on known svn repository.
Default policy:
Keep data as close as possible from the original svn data. We
only add information that are needed for update or continuing
from last known revision (svn revision and svn repository's
uuid).
"""
- def __init__(self):
- super().__init__()
-
def cleanup(self):
"""Clean after oneself.
This is in charge to flush the remaining data to write in swh storage.
And to clean up the svn repository's working representation on disk.
"""
self.svnrepo.clean_fs()
def swh_revision_hash_tree_at_svn_revision(self, revision):
"""Compute a given hash tree at specific revision.
"""
local_dirname, local_url = self.svnrepo.export_temporary(revision)
h = hashtree(local_url)['sha1_git']
self.svnrepo.clean_fs(local_dirname)
return h
def get_svn_repo(self, svn_url, destination_path, origin):
return svn.SWHSvnRepo(
svn_url, origin['id'], self.storage,
destination_path=destination_path)
def swh_previous_revision(self, prev_swh_revision=None):
"""Retrieve swh's previous revision if any.
"""
return self.svnrepo.swh_previous_revision(prev_swh_revision)
def check_history_not_altered(self, svnrepo, revision_start, swh_rev):
"""Given a svn repository, check if the history was not tampered with.
"""
revision_id = swh_rev['id']
parents = swh_rev['parents']
hash_data_per_revs = svnrepo.swh_hash_data_at_revision(revision_start)
rev = revision_start
rev, _, commit, objects_per_path = list(hash_data_per_revs)[0]
dir_id = objects_per_path[b'']['checksums']['sha1_git']
swh_revision = self.build_swh_revision(rev,
commit,
dir_id,
parents)
swh_revision_id = git.compute_revision_sha1_git(swh_revision)
return swh_revision_id == revision_id
def build_swh_revision(self, rev, commit, dir_id, parents):
"""Build the swh revision dictionary.
This adds:
- the `'synthetic`' flag to true
- the '`extra_headers`' containing the repository's uuid and the
svn revision number.
Args:
rev: the svn revision
commit: the commit metadata
dir_id: the upper tree's hash identifier
parents: the parents' identifiers
Returns:
The swh revision corresponding to the svn revision.
"""
return converters.build_swh_revision(rev,
commit,
self.svnrepo.uuid,
dir_id,
parents)
def init_from(self, partial_swh_revision, previous_swh_revision):
"""Function to determine from where to start from.
Args:
partial_swh_revision: A known revision from which
the previous loading did not finish.
known_previous_revision: A known revision from which the
previous loading did finish.
Returns:
The revision from which to start or None if nothing (fresh
start).
"""
if partial_swh_revision and not previous_swh_revision:
return partial_swh_revision
if not partial_swh_revision and previous_swh_revision:
return previous_swh_revision
if partial_swh_revision and previous_swh_revision:
# will determine from which to start from
extra_headers1 = dict(
partial_swh_revision['metadata']['extra_headers'])
extra_headers2 = dict(
previous_swh_revision['metadata']['extra_headers'])
rev_start1 = int(extra_headers1['svn_revision'])
rev_start2 = int(extra_headers2['svn_revision'])
if rev_start1 <= rev_start2:
return previous_swh_revision
return partial_swh_revision
return None
def process_repository(self, origin_visit, last_known_swh_revision=None):
svnrepo = self.svnrepo
# default configuration
revision_start = 1
revision_parents = {
revision_start: []
}
# Check if we already know a previous revision for that origin
swh_rev = self.swh_previous_revision()
# Determine from which known revision to start
swh_rev = self.init_from(last_known_swh_revision,
previous_swh_revision=swh_rev)
if swh_rev: # Yes, we do. Try and update it.
extra_headers = dict(swh_rev['metadata']['extra_headers'])
revision_start = int(extra_headers['svn_revision'])
revision_parents = {
revision_start: swh_rev['parents'],
}
self.log.debug('svn export --ignore-keywords %s@%s' % (
svnrepo.remote_url,
revision_start))
if swh_rev and not self.check_history_not_altered(
svnrepo,
revision_start,
swh_rev):
msg = 'History of svn %s@%s history modified. Skipping...' % (
svnrepo.remote_url, revision_start)
self.log.warn(msg)
- raise SvnLoaderHistoryAltered
+ raise SvnLoaderHistoryAltered(msg, *self.args)
else:
# now we know history is ok, we start at next revision
revision_start = revision_start + 1
# and the parent become the latest know revision for
# that repository
revision_parents[revision_start] = [swh_rev['id']]
revision_end = svnrepo.head_revision()
self.log.info('[revision_start-revision_end]: [%s-%s]' % (
revision_start, revision_end))
if revision_start > revision_end and revision_start is not 1:
- self.log.info('%s@%s already injected.' % (
- svnrepo.remote_url, revision_end))
- raise SvnLoaderUneventful
+ msg = '%s@%s already injected.' % (svnrepo.remote_url,
+ revision_end)
+ self.log.info(msg)
+ raise SvnLoaderUneventful(msg, *self.args)
self.log.info('Processing %s.' % svnrepo)
# process and store revision to swh (sent by by blocks of
# 'revision_packet_size')
return self.process_swh_revisions(
svnrepo, revision_start, revision_end, revision_parents)
class SWHSvnLoaderFromDumpArchive(SWHSvnLoader):
"""Load a svn repository from an archive (containing a dump).
"""
def __init__(self, archive_path):
super().__init__()
self.log.info('Archive to mount and load %s' % archive_path)
self.temp_dir, self.repo_path = init_svn_repo_from_archive_dump(
archive_path)
def cleanup(self):
super().cleanup()
if self.temp_dir and os.path.exists(self.temp_dir):
self.log.debug('Clean up temp directory %s for project %s' % (
self.temp_dir, os.path.basename(self.repo_path)))
shutil.rmtree(self.temp_dir)
diff --git a/swh/loader/svn/tests/test_loader.py b/swh/loader/svn/tests/test_loader.py
index b0d0e04..ebf193e 100644
--- a/swh/loader/svn/tests/test_loader.py
+++ b/swh/loader/svn/tests/test_loader.py
@@ -1,619 +1,621 @@
# Copyright (C) 2016-2017 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from nose.tools import istest
from swh.model import hashutil
from swh.loader.svn.loader import SWHSvnLoader
from swh.loader.svn.loader import SvnLoaderHistoryAltered, SvnLoaderUneventful
from test_base import BaseTestSvnLoader
# Define loaders with no storage
# They'll just accumulate the data in place
# Only for testing purposes.
class TestSvnLoader:
"""Mixin class to inhibit the persistence and keep in memory the data
sent for storage.
cf. SWHSvnLoaderNoStorage
"""
def __init__(self):
super().__init__()
# We don't want to persist any result in this test context
self.config['send_contents'] = False
self.config['send_directories'] = False
self.config['send_revisions'] = False
self.config['send_releases'] = False
self.config['send_occurrences'] = False
# Init the state
self.all_contents = []
self.all_directories = []
self.all_revisions = []
self.all_releases = []
self.all_occurrences = []
# Check at each svn revision that the hash tree computation
# does not diverge
self.check_revision = 1
def maybe_load_contents(self, all_contents):
self.all_contents.extend(all_contents)
def maybe_load_directories(self, all_directories):
self.all_directories.extend(all_directories)
def maybe_load_revisions(self, all_revisions):
self.all_revisions.extend(all_revisions)
def maybe_load_releases(self, releases):
raise ValueError('If called, the test must break.')
def maybe_load_occurrences(self, all_occurrences):
self.all_occurrences.extend(all_occurrences)
# Override to do nothing at the end
def close_failure(self):
pass
def close_success(self):
pass
# Override to only prepare the svn repository
def prepare(self, *args, **kwargs):
self.svnrepo = self.get_svn_repo(*args)
class SWHSvnLoaderNoStorage(TestSvnLoader, SWHSvnLoader):
"""An SWHSVNLoader with no persistence.
Context:
Load a new svn repository using the swh policy (so no update).
"""
def swh_previous_revision(self, prev_swh_revision=None):
"""We do not know this repository so no revision.
"""
return None
class SWHSvnLoaderUpdateNoStorage(TestSvnLoader, SWHSvnLoader):
"""An SWHSVNLoader with no persistence.
Context:
Load a known svn repository using the swh policy.
We can either:
- do nothing since it does not contain any new commit (so no
change)
- either check its history is not altered and update in
consequence by loading the new revision
"""
def swh_previous_revision(self, prev_swh_revision=None):
"""Avoid the storage persistence call and return the expected previous
revision for that repository.
Check the following for explanation about the hashes:
- test_loader.org for (swh policy).
- cf. SWHSvnLoaderITTest
"""
return {
'id': hashutil.hash_to_bytes(
'4876cb10aec6f708f7466dddf547567b65f6c39c'),
'parents': [hashutil.hash_to_bytes(
'a3a577948fdbda9d1061913b77a1588695eadb41')],
'directory': hashutil.hash_to_bytes(
'0deab3023ac59398ae467fc4bff5583008af1ee2'),
'target_type': 'revision',
'metadata': {
'extra_headers': [
['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'],
['svn_revision', '6']
]
}
}
class SWHSvnLoaderUpdateHistoryAlteredNoStorage(TestSvnLoader, SWHSvnLoader):
"""An SWHSVNLoader with no persistence.
Context: Load a known svn repository using the swh policy with its
history altered so we do not update it.
"""
def swh_previous_revision(self, prev_swh_revision=None):
"""Avoid the storage persistence call and return the expected previous
revision for that repository.
Check the following for explanation about the hashes:
- test_loader.org for (swh policy).
- cf. SWHSvnLoaderITTest
"""
return {
# Changed the revision id's hash to simulate history altered
'id': hashutil.hash_to_bytes(
'badbadbadbadf708f7466dddf547567b65f6c39d'),
'parents': [hashutil.hash_to_bytes(
'a3a577948fdbda9d1061913b77a1588695eadb41')],
'directory': hashutil.hash_to_bytes(
'0deab3023ac59398ae467fc4bff5583008af1ee2'),
'target_type': 'revision',
'metadata': {
'extra_headers': [
['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'],
['svn_revision', b'6']
]
}
}
class SWHSvnLoaderNewRepositoryITTest(BaseTestSvnLoader):
def setUp(self):
super().setUp()
self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'}
self.origin_visit = {
'origin': self.origin['id'],
'visit': 2,
}
self.loader = SWHSvnLoaderNoStorage()
self.loader.prepare(
self.svn_mirror_url, self.destination_path, self.origin)
@istest
def process_repository(self):
"""Process a new repository with swh policy should be ok.
"""
# when
self.loader.process_repository(self.origin_visit)
# then
self.assertEquals(len(self.loader.all_revisions), 6)
self.assertEquals(len(self.loader.all_releases), 0)
last_revision = '4876cb10aec6f708f7466dddf547567b65f6c39c'
# cf. test_loader.org for explaining from where those hashes
# come from
expected_revisions = {
# revision hash | directory hash
'0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71': '669a71cce6c424a81ba42b7dc5d560d32252f0ca', # noqa
'95edacc8848369d6fb1608e887d6d2474fd5224f': '008ac97a1118560797c50e3392fa1443acdaa349', # noqa
'fef26ea45a520071711ba2b9d16a2985ee837021': '3780effbe846a26751a95a8c95c511fb72be15b4', # noqa
'3f51abf3b3d466571be0855dfa67e094f9ceff1b': 'ffcca9b09c5827a6b8137322d4339c8055c3ee1e', # noqa
'a3a577948fdbda9d1061913b77a1588695eadb41': '7dc52cc04c3b8bd7c085900d60c159f7b846f866', # noqa
last_revision: '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa
}
for rev in self.loader.all_revisions:
rev_id = hashutil.hash_to_hex(rev['id'])
directory_id = hashutil.hash_to_hex(rev['directory'])
self.assertEquals(expected_revisions[rev_id], directory_id)
class SWHSvnLoaderUpdateWithNoChangeITTest(BaseTestSvnLoader):
def setUp(self):
super().setUp()
self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'}
self.origin_visit = {
'origin': self.origin['id'],
'visit': 3,
}
self.loader = SWHSvnLoaderUpdateNoStorage()
self.loader.prepare(
self.svn_mirror_url, self.destination_path, self.origin)
@istest
def process_repository(self):
"""Process a known repository with swh policy and no new data should
be ok.
"""
# when
with self.assertRaises(SvnLoaderUneventful):
+ self.loader.args = (self.origin_visit,)
self.loader.process_repository(self.origin_visit)
# then
self.assertEquals(len(self.loader.all_revisions), 0)
self.assertEquals(len(self.loader.all_releases), 0)
class SWHSvnLoaderUpdateWithHistoryAlteredITTest(BaseTestSvnLoader):
def setUp(self):
# the svn repository pkg-gourmet has been updated with changes
super().setUp(archive_name='pkg-gourmet-with-updates.tgz')
self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'}
self.origin_visit = {
'origin': self.origin['id'],
'visit': 4,
}
self.loader = SWHSvnLoaderUpdateHistoryAlteredNoStorage()
self.loader.prepare(
self.svn_mirror_url, self.destination_path, self.origin)
@istest
def process_repository(self):
- """Process a known repository with swh policy and history altered should
- stop and do nothing.
+ """Process a known repository with swh policy and history altered
+ should stop and do nothing.
"""
# when
with self.assertRaises(SvnLoaderHistoryAltered):
+ self.loader.args = (self.origin_visit,)
self.loader.process_repository(self.origin_visit)
# then
# we got the previous run's last revision (rev 6)
# so 2 news + 1 old
self.assertEquals(len(self.loader.all_revisions), 0)
self.assertEquals(len(self.loader.all_releases), 0)
class SWHSvnLoaderUpdateWithChangesITTest(BaseTestSvnLoader):
def setUp(self):
# the svn repository pkg-gourmet has been updated with changes
super().setUp(archive_name='pkg-gourmet-with-updates.tgz')
self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'}
self.origin_visit = {
'origin': self.origin['id'],
'visit': 5,
}
self.loader = SWHSvnLoaderUpdateNoStorage()
self.loader.prepare(
self.svn_mirror_url, self.destination_path, self.origin)
@istest
def process_repository(self):
"""Process a known repository with swh policy and new data should
yield new revisions and occurrence.
"""
# when
self.loader.process_repository(self.origin_visit)
# then
# we got the previous run's last revision (rev 6)
# so 2 new
self.assertEquals(len(self.loader.all_revisions), 5)
self.assertEquals(len(self.loader.all_releases), 0)
last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a'
# cf. test_loader.org for explaining from where those hashes
# come from
expected_revisions = {
# revision hash | directory hash
'7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa
'38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa
'99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa
'902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa
last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa
}
for rev in self.loader.all_revisions:
rev_id = hashutil.hash_to_hex(rev['id'])
directory_id = hashutil.hash_to_hex(rev['directory'])
self.assertEquals(expected_revisions[rev_id], directory_id)
class SWHSvnLoaderUpdateWithUnfinishedLoadingChangesITTest(BaseTestSvnLoader):
def setUp(self):
super().setUp(archive_name='pkg-gourmet-with-updates.tgz')
self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'}
self.origin_visit = {
'origin': self.origin['id'],
'visit': 6
}
self.loader = SWHSvnLoaderNoStorage()
self.loader.prepare(
self.svn_mirror_url, self.destination_path, self.origin)
@istest
def process_repository(self):
"""Process a known repository with swh policy, the previous run did
not finish, so this finishes the loading
"""
previous_unfinished_revision = {
'id': hashutil.hash_to_bytes(
'4876cb10aec6f708f7466dddf547567b65f6c39c'),
'parents': [hashutil.hash_to_bytes(
'a3a577948fdbda9d1061913b77a1588695eadb41')],
'directory': hashutil.hash_to_bytes(
'0deab3023ac59398ae467fc4bff5583008af1ee2'),
'target_type': 'revision',
'metadata': {
'extra_headers': [
['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'],
['svn_revision', '6']
]
}
}
# when
self.loader.process_repository(
self.origin_visit,
last_known_swh_revision=previous_unfinished_revision)
# then
# we got the previous run's last revision (rev 6)
# so 2 new
self.assertEquals(len(self.loader.all_revisions), 5)
self.assertEquals(len(self.loader.all_releases), 0)
last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a'
# cf. test_loader.org for explaining from where those hashes
# come from
expected_revisions = {
# revision hash | directory hash
'7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa
'38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa
'99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa
'902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa
last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa
}
for rev in self.loader.all_revisions:
rev_id = hashutil.hash_to_hex(rev['id'])
directory_id = hashutil.hash_to_hex(rev['directory'])
self.assertEquals(expected_revisions[rev_id], directory_id)
class SWHSvnLoaderUpdateWithUnfinishedLoadingChangesButOccurrenceDoneITTest(
BaseTestSvnLoader):
def setUp(self):
super().setUp(archive_name='pkg-gourmet-with-updates.tgz')
self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'}
self.origin_visit = {
'origin': self.origin['id'],
'visit': 9,
}
self.loader = SWHSvnLoaderUpdateNoStorage()
self.loader.prepare(
self.svn_mirror_url, self.destination_path, self.origin)
@istest
def process_repository(self):
"""known repository, swh policy, unfinished revision is less recent
than occurrence, we start from last occurrence.
"""
previous_unfinished_revision = {
'id': hashutil.hash_to_bytes(
'a3a577948fdbda9d1061913b77a1588695eadb41'),
'parents': [hashutil.hash_to_bytes(
'3f51abf3b3d466571be0855dfa67e094f9ceff1b')],
'directory': hashutil.hash_to_bytes(
'7dc52cc04c3b8bd7c085900d60c159f7b846f866'),
'target_type': 'revision',
'metadata': {
'extra_headers': [
['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'],
['svn_revision', '5']
]
}
}
# when
self.loader.process_repository(
self.origin_visit,
last_known_swh_revision=previous_unfinished_revision)
# then
# we got the previous run's last revision (rev 6)
# so 2 new
self.assertEquals(len(self.loader.all_revisions), 5)
self.assertEquals(len(self.loader.all_releases), 0)
last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a'
# cf. test_loader.org for explaining from where those hashes
# come from
expected_revisions = {
# revision hash | directory hash
'7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa
'38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa
'99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa
'902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa
last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa
}
for rev in self.loader.all_revisions:
rev_id = hashutil.hash_to_hex(rev['id'])
directory_id = hashutil.hash_to_hex(rev['directory'])
self.assertEquals(expected_revisions[rev_id], directory_id)
class SWHSvnLoaderUpdateLessRecentNoStorage(TestSvnLoader, SWHSvnLoader):
"""An SWHSVNLoader with no persistence.
Context:
Load a known svn repository using the swh policy.
The last occurrence seen is less recent than a previous
unfinished crawl.
"""
def swh_previous_revision(self, prev_swh_revision=None):
"""Avoid the storage persistence call and return the expected previous
revision for that repository.
Check the following for explanation about the hashes:
- test_loader.org for (swh policy).
- cf. SWHSvnLoaderITTest
"""
return {
'id': hashutil.hash_to_bytes(
'a3a577948fdbda9d1061913b77a1588695eadb41'),
'parents': [hashutil.hash_to_bytes(
'3f51abf3b3d466571be0855dfa67e094f9ceff1b')],
'directory': hashutil.hash_to_bytes(
'7dc52cc04c3b8bd7c085900d60c159f7b846f866'),
'target_type': 'revision',
'metadata': {
'extra_headers': [
['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'],
['svn_revision', '5']
]
}
}
class SWHSvnLoaderUnfinishedLoadingChangesSinceLastOccurrenceITTest(
BaseTestSvnLoader):
def setUp(self):
super().setUp(archive_name='pkg-gourmet-with-updates.tgz')
self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'}
self.origin_visit = {
'origin': self.origin['id'],
'visit': 1,
}
self.loader = SWHSvnLoaderUpdateLessRecentNoStorage()
self.loader.prepare(
self.svn_mirror_url, self.destination_path, self.origin)
@istest
def process_repository(self):
"""known repository, swh policy, unfinished revision is less recent
than occurrence, we start from last occurrence.
"""
previous_unfinished_revision = {
'id': hashutil.hash_to_bytes(
'4876cb10aec6f708f7466dddf547567b65f6c39c'),
'parents': [hashutil.hash_to_bytes(
'a3a577948fdbda9d1061913b77a1588695eadb41')],
'directory': hashutil.hash_to_bytes(
'0deab3023ac59398ae467fc4bff5583008af1ee2'),
'target_type': 'revision',
'metadata': {
'extra_headers': [
['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'],
['svn_revision', '6']
]
}
}
# when
self.loader.process_repository(
self.origin_visit,
last_known_swh_revision=previous_unfinished_revision)
# then
# we got the previous run's last revision (rev 6)
# so 2 new
self.assertEquals(len(self.loader.all_revisions), 5)
self.assertEquals(len(self.loader.all_releases), 0)
last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a'
# cf. test_loader.org for explaining from where those hashes
# come from
expected_revisions = {
# revision hash | directory hash
'7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa
'38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa
'99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa
'902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa
last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa
}
for rev in self.loader.all_revisions:
rev_id = hashutil.hash_to_hex(rev['id'])
directory_id = hashutil.hash_to_hex(rev['directory'])
self.assertEquals(expected_revisions[rev_id], directory_id)
class SWHSvnLoaderUpdateAndTestCornerCasesAboutEolITTest(BaseTestSvnLoader):
def setUp(self):
super().setUp(archive_name='pkg-gourmet-with-eol-corner-cases.tgz')
self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'}
self.origin_visit = {
'origin': self.origin['id'],
'visit': 1,
}
self.loader = SWHSvnLoaderUpdateLessRecentNoStorage()
self.loader.prepare(
self.svn_mirror_url, self.destination_path, self.origin)
@istest
def process_repository(self):
"""EOL corner cases and update.
"""
previous_unfinished_revision = {
'id': hashutil.hash_to_bytes(
'171dc35522bfd17dda4e90a542a0377fb2fc707a'),
'parents': [hashutil.hash_to_bytes(
'902f29b4323a9b9de3af6d28e72dd581e76d9397')],
'directory': hashutil.hash_to_bytes(
'fd24a76c87a3207428e06612b49860fc78e9f6dc'),
'target_type': 'revision',
'metadata': {
'extra_headers': [
['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'],
['svn_revision', '11']
]
}
}
# when
self.loader.process_repository(
self.origin_visit,
last_known_swh_revision=previous_unfinished_revision)
# then
# we got the previous run's last revision (rev 11)
# so 8 new
self.assertEquals(len(self.loader.all_revisions), 8)
self.assertEquals(len(self.loader.all_releases), 0)
last_revision = '0148ae3eaa520b73a50802c59f3f416b7a36cf8c'
# cf. test_loader.org for explaining from where those hashes
# come from
expected_revisions = {
# revision hash | directory hash
'027e8769f4786597436ab94a91f85527d04a6cbb': '2d9ca72c6afec6284fb01e459588cbb007017c8c', # noqa
'4474d96018877742d9697d5c76666c9693353bfc': 'ab111577e0ab39e4a157c476072af48f2641d93f', # noqa
'97ad21eab92961e2a22ca0285f09c6d1e9a7ffbc': 'ab111577e0ab39e4a157c476072af48f2641d93f', # noqa
'd04ea8afcee6205cc8384c091bfc578931c169fd': 'b0a648b02e55a4dce356ac35187a058f89694ec7', # noqa
'ded78810401fd354ffe894aa4a1e5c7d30a645d1': 'b0a648b02e55a4dce356ac35187a058f89694ec7', # noqa
'4ee95e39358712f53c4fc720da3fafee9249ed19': 'c3c98df624733fef4e592bef983f93e2ed02b179', # noqa
'ffa901b69ca0f46a2261f42948838d19709cb9f8': 'c3c98df624733fef4e592bef983f93e2ed02b179', # noqa
last_revision: '844d4646d6c2b4f3a3b2b22ab0ee38c7df07bab2', # noqa
}
for rev in self.loader.all_revisions:
rev_id = hashutil.hash_to_hex(rev['id'])
directory_id = hashutil.hash_to_hex(rev['directory'])
self.assertEquals(expected_revisions[rev_id], directory_id)

File Metadata

Mime Type
text/x-diff
Expires
Mon, Aug 18, 11:36 PM (1 w, 14 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3336330

Event Timeline

About · Developer information · Legal