Page MenuHomeSoftware Heritage
Files F9696959

No OneTemporary
Actions

View Options

diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py
index 0bdcef2..db2bfd9 100644
--- a/swh/loader/svn/loader.py
+++ b/swh/loader/svn/loader.py
@@ -1,464 +1,466 @@
# Copyright (C) 2015-2016 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
"""Loader in charge of injecting either new or existing svn mirrors to
swh-storage.
"""
import datetime
from swh.core import utils
from swh.model import git, hashutil
from swh.model.git import GitType
from swh.loader.core.loader import SWHLoader
from . import svn, converters
class SvnLoaderException(ValueError):
"""A wrapper exception to transit the swh_revision onto which the
loading failed.
"""
def __init__(self, e, swh_revision):
super().__init__(e)
self.swh_revision = swh_revision
class BaseSvnLoader(SWHLoader):
"""Base Svn loader to load one svn repository.
There exists 2 different policies:
- git-svn one (not for production): cf. GitSvnSvnLoader
- SWH one: cf. SWHSvnLoader
The main entry point of this is (no need to override it)
- def load(self):
Inherit this class and then override the following functions:
- def build_swh_revision(self, rev, commit, dir_id, parents)
This is in charge of converting an svn revision to a compliant
swh revision
- def process_repository(self)
This is in charge of processing the actual svn repository and
store the result to swh storage.
"""
CONFIG_BASE_FILENAME = 'loader/svn.ini'
def __init__(self, svn_url, destination_path, origin):
super().__init__(origin['id'],
logging_class='swh.loader.svn.SvnLoader')
self.origin = origin
def build_swh_revision(self, rev, commit, dir_id, parents):
"""Convert an svn revision to an swh one according to the loader's
policy (git-svn or swh).
Args:
rev: the svn revision number
commit: dictionary with keys 'author_name', 'author_date', 'rev',
'message'
dir_id: the hash tree computation
parents: the revision's parents
Returns:
The swh revision
"""
raise NotImplementedError('This should be overriden by subclass')
def process_repository(self):
"""The main idea of this function is to:
- iterate over the svn commit logs
- extract the svn commit log metadata
- compute the hashes from the current directory down to the
file
- compute the equivalent swh revision
- send all those objects for storage
- create an swh occurrence pointing to the last swh revision
seen
- send that occurrence for storage in swh-storage.
"""
raise NotImplementedError('This should be implemented in subclass.')
def process_svn_revisions(self, svnrepo, revision_start, revision_end,
revision_parents):
"""Process revisions from revision_start to revision_end and send to
swh for storage.
At each svn revision, checkout the repository, compute the
tree hash and blobs and send for swh storage to store.
Then computes and yields the swh revision.
Yields:
swh revision
"""
gen_revs = svnrepo.swh_hash_data_per_revision(
revision_start,
revision_end)
swh_revision = None
for rev, nextrev, commit, objects_per_path in gen_revs:
# Send the associated contents/directories
self.maybe_load_contents(
git.objects_per_type(GitType.BLOB, objects_per_path))
self.maybe_load_directories(
git.objects_per_type(GitType.TREE, objects_per_path))
# compute the fs tree's checksums
dir_id = objects_per_path[b'']['checksums']['sha1_git']
swh_revision = self.build_swh_revision(
rev, commit, dir_id, revision_parents[rev])
swh_revision['id'] = git.compute_revision_sha1_git(
swh_revision)
self.log.debug('rev: %s, swhrev: %s, dir: %s' % (
rev,
hashutil.hash_to_hex(swh_revision['id']),
hashutil.hash_to_hex(dir_id)))
if nextrev:
revision_parents[nextrev] = [swh_revision['id']]
yield swh_revision
def process_swh_revisions(self,
svnrepo,
revision_start,
revision_end,
revision_parents):
"""Process and store revision to swh (sent by blocks of
'revision_packet_size')
Returns:
The latest revision stored.
"""
try:
swh_revision_gen = self.process_svn_revisions(svnrepo,
revision_start,
revision_end,
revision_parents)
revs = []
for revisions in utils.grouper(
swh_revision_gen,
self.config['revision_packet_size']):
revs = list(revisions)
self.log.info('Processed %s revisions: [%s, ...]' % (
len(revs), hashutil.hash_to_hex(revs[0]['id'])))
self.maybe_load_revisions(revs)
except Exception as e:
if revs:
self.maybe_load_revisions(revs)
know_swh_rev = revs[-1]
# Wrap the exception with the needed revision
raise SvnLoaderException(e, swh_revision={
'id': know_swh_rev['id'],
'parents': know_swh_rev['parents'],
'metadata': know_swh_rev.get('metadata')
})
else:
raise SvnLoaderException(e, swh_revision=None)
return revs[-1]
def process_swh_occurrence(self, revision, origin):
"""Process and load the occurrence pointing to the latest revision.
"""
occ = converters.build_swh_occurrence(revision['id'],
origin['id'],
datetime.datetime.utcnow())
self.log.debug('occ: %s' % occ)
self.maybe_load_occurrences([occ])
def load(self, known_state=None):
"""Load a svn repository in swh.
Checkout the svn repository locally in destination_path.
Args:
- svn_url: svn repository url to import
- origin: Dictionary origin
- id: origin's id
- url: url origin we fetched
- type: type of the origin
Returns:
Dictionary with the following keys:
- status: mandatory, the status result as a boolean
- stderr: optional when status is True, mandatory otherwise
"""
try:
self.process_repository(known_state)
finally:
# flush eventual remaining data
self.flush()
self.svnrepo.clean_fs()
return {'status': True}
class GitSvnSvnLoader(BaseSvnLoader):
"""Git-svn like loader (compute hashes a-la git-svn)
Notes:
This implementation is:
- NOT for production
- NOT able to deal with update.
Default policy:
Its default policy is to enrich (or even alter) information at
each svn revision. It will:
- truncate the timestamp of the svn commit date
- alter the user to be an email using the repository's uuid as
mailserver (user -> user@<repo-uuid>)
- fills in the gap for empty author with '(no author)' name
- remove empty folder (thus not counting them during hash computation)
The equivalent git command is: `git svn clone <repo-url> -q
--no-metadata`
"""
- def __init__(self, svn_url, destination_path, origin):
+ def __init__(self, svn_url, destination_path, origin, svn_uuid=None):
super().__init__(svn_url, destination_path, origin)
# We don't want to persist result in git-svn policy
self.config['send_contents'] = False
self.config['send_directories'] = False
self.config['send_revisions'] = False
self.config['send_releases'] = False
self.config['send_occurrences'] = False
self.svnrepo = svn.GitSvnSvnRepo(
svn_url,
origin['id'],
self.storage,
- destination_path=destination_path)
+ destination_path=destination_path,
+ svn_uuid=svn_uuid)
def build_swh_revision(self, rev, commit, dir_id, parents):
"""Build the swh revision a-la git-svn.
Args:
rev: the svn revision
commit: the commit metadata
dir_id: the upper tree's hash identifier
parents: the parents' identifiers
Returns:
The swh revision corresponding to the svn revision
without any extra headers.
"""
return converters.build_gitsvn_swh_revision(rev,
commit,
dir_id,
parents)
def process_repository(self, known_state=None):
"""Load the repository's svn commits and process them as swh hashes.
This does not:
- deal with update
- nor with the potential known state.
"""
origin = self.origin
svnrepo = self.svnrepo
# default configuration
revision_start = 1
revision_parents = {
revision_start: []
}
revision_end = svnrepo.head_revision()
self.log.info('[revision_start-revision_end]: [%s-%s]' % (
revision_start, revision_end))
if revision_start == revision_end and revision_start is not 1:
self.log.info('%s@%s already injected.' % (
svnrepo.remote_url, revision_end))
return {'status': True}
self.log.info('Processing %s.' % svnrepo)
# process and store revision to swh (sent by by blocks of
# 'revision_packet_size')
latest_rev = self.process_swh_revisions(svnrepo,
revision_start,
revision_end,
revision_parents)
self.process_swh_occurrence(latest_rev, origin)
class SWHSvnLoader(BaseSvnLoader):
"""Swh svn loader is the main implementation destined for production.
This implementation is able to deal with update on known svn repository.
Default policy:
It's to not add any information and be as close as possible
from the svn data the server sent its way.
The only thing that are added are the swh's revision
'extra_header' to be able to deal with update.
"""
- def __init__(self, svn_url, destination_path, origin):
+ def __init__(self, svn_url, destination_path, origin, svn_uuid=None):
super().__init__(svn_url, destination_path, origin)
self.svnrepo = svn.SWHSvnRepo(
svn_url,
origin['id'],
self.storage,
- destination_path=destination_path)
+ destination_path=destination_path,
+ svn_uuid=svn_uuid)
def swh_previous_revision(self):
"""Retrieve swh's previous revision if any.
"""
return self.svnrepo.swh_previous_revision()
def check_history_not_altered(self, svnrepo, revision_start, swh_rev):
"""Given a svn repository, check if the history was not tampered with.
"""
revision_id = swh_rev['id']
parents = swh_rev['parents']
hash_data_per_revs = svnrepo.swh_hash_data_at_revision(revision_start)
rev = revision_start
rev, _, commit, objects_per_path = list(hash_data_per_revs)[0]
dir_id = objects_per_path[b'']['checksums']['sha1_git']
swh_revision = self.build_swh_revision(rev,
commit,
dir_id,
parents)
swh_revision_id = git.compute_revision_sha1_git(swh_revision)
return swh_revision_id == revision_id
def build_swh_revision(self, rev, commit, dir_id, parents):
"""Build the swh revision dictionary.
This adds:
- the 'synthetic' flag to true
- the 'extra_headers' containing the repository's uuid and the
svn revision number.
Args:
rev: the svn revision
commit: the commit metadata
dir_id: the upper tree's hash identifier
parents: the parents' identifiers
Returns:
The swh revision corresponding to the svn revision.
"""
return converters.build_swh_revision(rev,
commit,
self.svnrepo.uuid,
dir_id,
parents)
def init_from(self, partial_swh_revision, previous_swh_revision):
"""Function to determine from where to start from.
Args:
- partial_swh_revision: A known revision from which
the previous loading did not finish.
- known_previous_revision: A known revision from which the
previous loading did finish.
Returns:
The revision from which to start or None if nothing (fresh
start).
"""
if partial_swh_revision and not previous_swh_revision:
return partial_swh_revision
if not partial_swh_revision and previous_swh_revision:
return previous_swh_revision
if partial_swh_revision and previous_swh_revision:
# will determine from which to start from
extra_headers1 = dict(
partial_swh_revision['metadata']['extra_headers'])
extra_headers2 = dict(
previous_swh_revision['metadata']['extra_headers'])
rev_start1 = int(extra_headers1['svn_revision'])
rev_start2 = int(extra_headers2['svn_revision'])
if rev_start1 <= rev_start2:
return previous_swh_revision
return partial_swh_revision
return None
def process_repository(self, known_state=None):
svnrepo = self.svnrepo
origin = self.origin
# default configuration
revision_start = 1
revision_parents = {
revision_start: []
}
# Check if we already know a previous revision for that origin
swh_rev = self.swh_previous_revision()
# Determine from which known revision to start
swh_rev = self.init_from(known_state, previous_swh_revision=swh_rev)
if swh_rev: # Yes, we do. Try and update it.
extra_headers = dict(swh_rev['metadata']['extra_headers'])
revision_start = int(extra_headers['svn_revision'])
revision_parents = {
revision_start: swh_rev['parents'],
}
self.log.debug('svn export --ignore-keywords %s@%s' % (
svnrepo.remote_url,
revision_start))
if swh_rev and not self.check_history_not_altered(
svnrepo,
revision_start,
swh_rev):
msg = 'History of svn %s@%s history modified. Skipping...' % ( # noqa
svnrepo.remote_url, revision_start)
self.log.warn(msg)
return {'status': False, 'stderr': msg}
else:
# now we know history is ok, we start at next revision
revision_start = revision_start + 1
# and the parent become the latest know revision for
# that repository
revision_parents[revision_start] = [swh_rev['id']]
revision_end = svnrepo.head_revision()
self.log.info('[revision_start-revision_end]: [%s-%s]' % (
revision_start, revision_end))
if revision_start >= revision_end and revision_start is not 1:
self.log.info('%s@%s already injected.' % (
svnrepo.remote_url, revision_end))
return {'status': True}
self.log.info('Processing %s.' % svnrepo)
# process and store revision to swh (sent by by blocks of
# 'revision_packet_size')
latest_rev = self.process_swh_revisions(svnrepo,
revision_start,
revision_end,
revision_parents)
self.process_swh_occurrence(latest_rev, origin)
diff --git a/swh/loader/svn/producer.py b/swh/loader/svn/producer.py
index 5a0cec9..75315cb 100644
--- a/swh/loader/svn/producer.py
+++ b/swh/loader/svn/producer.py
@@ -1,43 +1,61 @@
# Copyright (C) 2015-2016 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import click
import sys
task_name = 'swh.loader.svn.tasks.LoadSvnRepositoryTsk'
-def libproduce(svn_url, destination_path=None, synchroneous=False):
+def libproduce(svn_url, original_svn_url, original_svn_uuid,
+ destination_path=None, synchroneous=False):
from swh.scheduler.celery_backend.config import app
for module in app.conf.CELERY_IMPORTS:
__import__(module)
task = app.tasks[task_name]
if not synchroneous and svn_url:
- task.delay(svn_url=svn_url, destination_path=destination_path)
+ task.delay(svn_url=svn_url,
+ original_svn_url=original_svn_url,
+ original_svn_uuid=original_svn_uuid,
+ destination_path=destination_path)
elif synchroneous and svn_url: # for debug purpose
- task(svn_url=svn_url, destination_path=destination_path)
+ task(svn_url=svn_url,
+ original_svn_url=original_svn_url,
+ original_svn_uuid=original_svn_uuid,
+ destination_path=destination_path)
else: # synchroneous flag is ignored in that case
- for svn_url in sys.stdin:
+ for args in sys.stdin:
svn_url = svn_url.rstrip()
print(svn_url)
- task.delay(svn_url=svn_url, destination_path=destination_path)
+ task.delay(svn_url=svn_url,
+ original_svn_url=original_svn_url,
+ original_svn_uuid=original_svn_uuid,
+ destination_path=destination_path)
@click.command()
@click.option('--svn-url',
- help="svn repository's remote url.")
+ help="svn repository's mirror url.")
+@click.option('--original-svn-url', default=None,
+ help='svn repository\'s original remote url '
+ '(if different than --svn-url).')
+@click.option('--original-svn-uuid', default=None,
+ help='svn repository\'s original uuid '
+ '(to provide when using --original-svn-url)')
@click.option('--destination-path',
help="(optional) svn checkout destination.")
@click.option('--synchroneous',
is_flag=True,
help="To execute directly the svn loading.")
-def produce(svn_url, destination_path, synchroneous):
- libproduce(svn_url, destination_path, synchroneous)
+def produce(svn_url, original_svn_url, original_svn_uuid, destination_path,
+ synchroneous):
+ libproduce(svn_url, original_svn_url, original_svn_uuid, destination_path,
+ synchroneous)
if __name__ == '__main__':
produce()
diff --git a/swh/loader/svn/svn.py b/swh/loader/svn/svn.py
index aac8ea4..554ce73 100644
--- a/swh/loader/svn/svn.py
+++ b/swh/loader/svn/svn.py
@@ -1,395 +1,406 @@
# Copyright (C) 2015-2016 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
"""SVN client in charge of iterating over svn logs and yield commit
representations including the hash tree/content computations per
svn commit.
"""
import os
import tempfile
import shutil
from subvertpy.ra import RemoteAccess, Auth, get_username_provider
from subvertpy import client, properties
from swh.model import git
from . import ra, utils, converters
# When log message contains empty data
DEFAULT_AUTHOR_MESSAGE = ''
class SvnRepoException(ValueError):
def __init__(self, svnrepo, e):
super().__init__(e)
self.svnrepo = svnrepo
class BaseSvnRepo():
"""Base svn repository representation for swh.
To override some of the behavior regarding the message log
properties, you can instantiate a subclass of this class and
override:
- def convert_commit_author(self, author)
- def convert_commit_message(self, msg)
- def convert_commit_date(self, date)
cf. GitSvnSvnRepo, SwhSvnRepo for instanciation example.
"""
def __init__(self, remote_url, origin_id, storage,
- destination_path=None):
+ destination_path=None,
+ svn_uuid=None):
self.remote_url = remote_url.rstrip('/')
self.storage = storage
self.origin_id = origin_id
if destination_path:
os.makedirs(destination_path, exist_ok=True)
root_dir = destination_path
else:
root_dir = '/tmp'
self.local_dirname = tempfile.mkdtemp(suffix='.swh.loader',
prefix='tmp.',
dir=root_dir)
local_name = os.path.basename(self.remote_url)
auth = Auth([get_username_provider()])
# one connection for log iteration
self.conn_log = RemoteAccess(self.remote_url,
auth=auth)
# another for replay
self.conn = RemoteAccess(self.remote_url,
auth=auth)
# one client for update operation
self.client = client.Client(auth=auth)
self.local_url = os.path.join(self.local_dirname, local_name).encode(
'utf-8')
- self.uuid = self.conn.get_uuid().encode('utf-8')
+
+ if svn_uuid:
+ self.uuid = svn_uuid.encode('utf-8')
+ else:
+ self.uuid = self.conn.get_uuid().encode('utf-8')
def __str__(self):
return str({'remote_url': self.remote_url,
'local_url': self.local_url,
'uuid': self.uuid,
'swh-origin': self.origin_id})
def head_revision(self):
"""Retrieve current revision of the repository's working copy.
"""
return self.conn.get_latest_revnum()
def initial_revision(self):
"""Retrieve the initial revision from which the remote url appeared.
Note: This should always be 1 since we won't be dealing with in-depth
url.
"""
return 1
def convert_commit_message(self, msg):
"""Do something with message (e.g add extra line, etc...)
cf. SvnRepo for a simple implementation.
Args:
msg (str): the commit message to convert.
Returns:
The transformed message as bytes.
"""
raise NotImplementedError('Should be overridden by subclass.')
def convert_commit_date(self, date):
"""Convert the message date (e.g, convert into timestamp or whatever
makes sense to you.).
Args:
date (str): the commit date to convert.
Returns:
The transformed date.
"""
raise NotImplementedError('Should be overridden by subclass.')
def convert_commit_author(self, author):
"""Convert the commit author (e.g, convert into dict or whatever
makes sense to you.).
Args:
author (str): the commit author to convert.
Returns:
The transformed author as dict.
"""
raise NotImplementedError('Should be overridden by subclass.')
def __to_entry(self, log_entry):
changed_paths, rev, revprops, has_children = log_entry
author_date = self.convert_commit_date(
revprops.get(properties.PROP_REVISION_DATE))
author = self.convert_commit_author(
revprops.get(properties.PROP_REVISION_AUTHOR))
message = self.convert_commit_message(
revprops.get(properties.PROP_REVISION_LOG,
DEFAULT_AUTHOR_MESSAGE))
return {
'rev': rev,
'author_date': author_date,
'author_name': author,
'message': message,
}
def logs(self, revision_start, revision_end):
"""Stream svn logs between revision_start and revision_end by chunks of
block_size logs.
Yields revision and associated revision information between the
revision start and revision_end.
Args:
revision_start: the svn revision starting bound
revision_end: the svn revision ending bound
Yields:
tuple of revisions and logs.
revisions: list of revisions in order
logs: Dictionary with key revision number and value the log entry.
The log entry is a dictionary with the following keys:
- author_date: date of the commit
- author_name: name of the author
- message: commit message
"""
for log_entry in self.conn_log.iter_log(paths=None,
start=revision_start,
end=revision_end,
discover_changed_paths=False):
yield self.__to_entry(log_entry)
def export(self, revision):
"""Export the repository to a given version.
"""
self.client.export(self.remote_url,
to=self.local_url.decode('utf-8'),
rev=revision)
def swh_previous_revision(self):
"""Look for possible existing revision in swh.
Returns:
The previous swh revision if found, None otherwise.
"""
storage = self.storage
occ = storage.occurrence_get(self.origin_id)
if occ:
revision_id = occ[0]['target']
revisions = storage.revision_get([revision_id])
if revisions:
return revisions[0]
def swh_hash_data_per_revision(self, start_revision, end_revision):
"""Compute swh hash data per each revision between start_revision and
end_revision.
Args:
start_revision: starting revision
end_revision: ending revision
Yields:
tuple (rev, nextrev, commit, objects_per_path)
- rev: current revision
- nextrev: next revision
- commit: commit data (author, date, message) for such revision
- objects_per_path: dictionary of path, swh hash data with type
"""
hashes = {}
for commit in self.logs(start_revision, end_revision):
rev = commit['rev']
hashes = self.swhreplay.compute_hashes(rev)
if rev == end_revision:
nextrev = None
else:
nextrev = rev + 1
yield rev, nextrev, commit, hashes
def swh_hash_data_at_revision(self, revision):
"""Compute the hash data at revision.
Expected to be used for update only.
"""
# Update the disk at revision
self.export(revision)
# Compute the current hashes on disk
hashes = git.compute_hashes_from_directory(self.local_url)
hashes = utils.convert_hashes_with_relative_path(
hashes,
rootpath=self.local_url)
# Update the replay collaborator with the right state
self.swhreplay = ra.SWHReplay(
- conn=self.conn,
- rootpath=self.local_url,
- objects=hashes)
+ conn=self.conn,
+ rootpath=self.local_url,
+ objects=hashes)
# Retrieve the commit information for revision
commit = list(self.logs(revision, revision))[0]
yield revision, revision + 1, commit, hashes
def clean_fs(self):
"""Clean up the local working copy.
"""
shutil.rmtree(self.local_dirname)
class GitSvnSvnRepo(BaseSvnRepo):
"""Svn repository mapping a-la git-svn.
This class does exactly as BaseSvnRepo except for:
- the commit message which is extended with a new line and then encoded
- the commit author is converted using the repository's uuid for his/her
email
- the commit date is transformed into timestamp and truncated
- Extra commit line in commit message.
- user@<uuid> in raw commit message
- truncated timestamp in raw commit message
"""
def __init__(self, remote_url, origin_id, storage,
- destination_path=None):
- super().__init__(remote_url, origin_id, storage, destination_path)
+ destination_path=None,
+ svn_uuid=None):
+ super().__init__(remote_url, origin_id, storage,
+ destination_path=destination_path,
+ svn_uuid=svn_uuid)
self.swhreplay = ra.SWHReplayNoEmptyFolder(
conn=self.conn,
rootpath=self.local_url)
self.with_empty_folder = False
def convert_commit_message(self, msg):
"""Add an extra line to the commit message and encode it in utf-8.
Args:
msg (str): the commit message to convert.
Returns:
The transformed message as bytes.
"""
return ('%s\n' % msg).encode('utf-8')
def convert_commit_date(self, date):
"""Convert the commit message date into truncated timestamp in swh
format.
Args:
date (str): the commit date to convert.
Returns:
The transformed date.
"""
return converters.svn_date_to_gitsvn_date(date)
def convert_commit_author(self, author):
"""Convert the commit author into an swh person.
The user becomes a dictionary of the form:
{
name: author,
email: author@repo-uuid,
fullname: author <author@repo-uuid>
}
If the user is already some kind of fullname, this is what is
used as fullname.
Args:
author (str): the commit author to convert.
Returns:
The transformed author as dict.
"""
return converters.svn_author_to_gitsvn_person(author, self.uuid)
class SWHSvnRepo(BaseSvnRepo):
"""This class does exactly as BaseSvnRepo except for:
- the commit message which is simply encoded
- the commit author is left as is.
- the commit timestamp is left as is.
"""
def __init__(self, remote_url, origin_id, storage,
- destination_path=None):
- super().__init__(remote_url, origin_id, storage, destination_path)
+ destination_path=None,
+ svn_uuid=None):
+ super().__init__(remote_url, origin_id, storage,
+ destination_path=destination_path,
+ svn_uuid=svn_uuid)
self.swhreplay = ra.SWHReplay(
conn=self.conn,
rootpath=self.local_url)
def convert_commit_message(self, msg):
"""Simply encode the commit message.
Args:
msg (str): the commit message to convert.
Returns:
The transformed message as bytes.
"""
return msg.encode('utf-8')
def convert_commit_date(self, date):
"""Convert the message commit date into a timestamp in swh format.
The precision is kept.
Args:
date (str): the commit date to convert.
Returns:
The transformed date.
"""
return converters.svn_date_to_swh_date(date)
def convert_commit_author(self, author):
"""Convert the commit author into an swh person.
The user becomes a dictionary of the form:
{
name: author,
email: '',
fullname: author
}
Args:
author (str): the commit author to convert.
Returns:
The transformed author as dict.
"""
return converters.svn_author_to_swh_person(author)
diff --git a/swh/loader/svn/tasks.py b/swh/loader/svn/tasks.py
index 414804a..59dbb90 100644
--- a/swh/loader/svn/tasks.py
+++ b/swh/loader/svn/tasks.py
@@ -1,94 +1,108 @@
# Copyright (C) 2015-2016 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from swh.loader.core import tasks
from .loader import GitSvnSvnLoader, SWHSvnLoader, SvnLoaderException
from .loader import converters
class LoadSvnRepositoryTsk(tasks.LoaderCoreTask):
"""Import one svn repository to Software Heritage.
"""
CONFIG_BASE_FILENAME = 'loader/svn.ini'
ADDITIONAL_CONFIG = {
'storage_class': ('str', 'remote_storage'),
'storage_args': ('list[str]', ['http://localhost:5000/']),
'with_policy': ('string', 'swh'), # Default, other possible
# value is 'gitsvn'
}
task_queue = 'swh_loader_svn'
def run(self, *args, **kwargs):
"""Import a svn repository.
Args:
- svn_url: svn's repository url
- destination_path: root directory to locally retrieve svn's data
- swh_revision: Optional extra swh revision to start from.
cf. swh.loader.svn.SvnLoader.process docstring
"""
destination_path = kwargs['destination_path']
+ # local svn url
svn_url = kwargs['svn_url']
+ # if original_svn_url is mentioned, this means we load a local mirror
+ original_svn_url = kwargs.get('original_svn_url', svn_url)
+ # potential uuid overwrite
+ original_svn_uuid = kwargs.get('original_svn_uuid')
+
+ if original_svn_url != svn_url and not original_svn_uuid:
+ msg = "When loading a local mirror, you must specify the original repository's uuid." # noqa
+ self.log.error('%s. Skipping mirror %s' % (msg, svn_url))
+ return
if 'origin' not in kwargs: # first time, we'll create the origin
origin = {
'type': 'svn',
- 'url': svn_url,
+ 'url': original_svn_url,
}
origin['id'] = self.storage.origin_add_one(origin)
retry = False
else:
origin = {
'id': kwargs['origin'],
- 'url': kwargs['svn_url'],
+ 'url': original_svn_url,
'type': 'svn'
}
retry = True
fetch_history_id = self.open_fetch_history(origin['id'])
try:
# Determine which loader to trigger
if self.config['with_policy'] == 'gitsvn':
# this one compute hashes but do not store anywhere
- loader = GitSvnSvnLoader(svn_url, destination_path, origin)
+ loader = GitSvnSvnLoader(svn_url, destination_path, origin,
+ svn_uuid=original_svn_uuid)
elif self.config['with_policy'] == 'swh':
# the real production use case with storage and all
- loader = SWHSvnLoader(svn_url, destination_path, origin)
+ loader = SWHSvnLoader(svn_url, destination_path, origin,
+ svn_uuid=original_svn_uuid)
else:
raise ValueError('Only gitsvn or swh policies are supported in'
'\'with_policy\' entry. '
'Please adapt your svn.ini file accordingly')
if retry:
swh_revision = converters.scheduler_to_loader_revision(
kwargs['swh_revision'])
result = loader.load(swh_revision)
else:
result = loader.load()
except SvnLoaderException as e:
# reschedule a task
swh_rev = converters.loader_to_scheduler_revision(e.swh_revision)
self.scheduler_backend.create_task({
'type': 'svn-loader',
'arguments': {
'args': None,
'kwargs': {
'origin': origin['id'],
'svn_url': svn_url,
+ 'original_svn_url': original_svn_url,
+ 'original_svn_uuid': original_svn_uuid,
'destination_path': destination_path,
'swh_revision': swh_rev,
}
}
})
self.log.error(
'Error during loading: %s - Svn repository rescheduled.' % e)
result = {'status': False}
self.close_fetch_history(fetch_history_id, result)

File Metadata

Mime Type
text/x-diff
Expires
Mon, Aug 18, 10:17 PM (5 d, 7 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3339723

Event Timeline

About · Developer information · Legal