Page MenuHomeSoftware Heritage
Files F9696279

No OneTemporary
Actions

View Options

diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py
index efee682..1750b2b 100644
--- a/swh/loader/svn/loader.py
+++ b/swh/loader/svn/loader.py
@@ -1,506 +1,506 @@
# Copyright (C) 2015-2017 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
"""Loader in charge of injecting either new or existing svn mirrors to
swh-storage.
"""
import abc
import os
import shutil
from swh.core import utils
from swh.model import git, hashutil
from swh.model.git import GitType
from swh.loader.core.loader import SWHLoader
from . import svn, converters
from .utils import hashtree, init_svn_repo_from_archive_dump
class SvnLoaderEventful(ValueError):
"""A wrapper exception to transit the swh_revision onto which the
loading failed.
"""
def __init__(self, e, swh_revision):
super().__init__(e)
self.swh_revision = swh_revision
class SvnLoaderUneventful(ValueError):
pass
class SvnLoaderHistoryAltered(ValueError):
pass
class BaseSvnLoader(SWHLoader, metaclass=abc.ABCMeta):
"""Base Svn loader to load one svn repository according to specific
policies (only swh one now).
The main entry point of this is (no need to override it)::
def load(self, origin_visit, last_known_swh_revision=None): pass
Inherit this class and then override the following functions::
def build_swh_revision(self, rev, commit, dir_id, parents):
# This is in charge of converting an svn revision to a compliant
# swh revision
def process_repository(self):
# This is in charge of processing the actual svn repository and
# store the result to swh storage.
"""
CONFIG_BASE_FILENAME = 'loader/svn'
ADDITIONAL_CONFIG = {
'check_revision': ('int', 1000),
}
def __init__(self):
super().__init__(logging_class='swh.loader.svn.SvnLoader')
self.check_revision = self.config['check_revision']
@abc.abstractmethod
def swh_revision_hash_tree_at_svn_revision(self, revision):
"""Compute and return the hash tree at a given svn revision.
Args:
rev (int): the svn revision we want to check
Returns:
The hash tree directory as bytes.
"""
pass
@abc.abstractmethod
def get_svn_repo(self, svn_url, destination_path, origin):
"""Instantiates the needed svnrepo collaborator to permit reading svn
repository.
Args:
svn_url: the svn repository url to read from
destination_path: the local path on disk to compute data
origin: the corresponding origin
Returns:
- Instance of :py:mod:`swh.loader.svn.svn` clients
+ Instance of :mod:`swh.loader.svn.svn` clients
"""
raise NotImplementedError
@abc.abstractmethod
def build_swh_revision(self, rev, commit, dir_id, parents):
"""Convert an svn revision to an swh one according to the loader's
policy (git-svn or swh).
Args:
rev: the svn revision number
commit: dictionary with keys: author\_name, author\_date, rev,
message
dir_id: the hash tree computation
parents: the revision's parents
Returns:
The swh revision
"""
raise NotImplementedError
@abc.abstractmethod
def process_repository(self, origin_visit, last_known_swh_revision=None):
"""The main idea of this function is to:
- iterate over the svn commit logs
- extract the svn commit log metadata
- compute the hashes from the current directory down to the file
- compute the equivalent swh revision
- send all those objects for storage
- create an swh occurrence pointing to the last swh revision seen
- send that occurrence for storage in swh-storage.
"""
raise NotImplementedError
def process_svn_revisions(self, svnrepo, revision_start, revision_end,
revision_parents):
"""Process revisions from revision_start to revision_end and send to
swh for storage.
At each svn revision, checkout the repository, compute the
tree hash and blobs and send for swh storage to store.
Then computes and yields the swh revision.
Note that at every self.check_revision, an svn export is done
and a hash tree is computed to check that no divergence
occurred.
Yields:
swh revision as a dictionary with keys, sha1_git, sha1, etc...
"""
gen_revs = svnrepo.swh_hash_data_per_revision(
revision_start,
revision_end)
swh_revision = None
count = 0
for rev, nextrev, commit, objects_per_path in gen_revs:
count += 1
# Send the associated contents/directories
self.maybe_load_contents(
git.objects_per_type(GitType.BLOB, objects_per_path))
self.maybe_load_directories(
git.objects_per_type(GitType.TREE, objects_per_path))
# compute the fs tree's checksums
dir_id = objects_per_path[b'']['checksums']['sha1_git']
swh_revision = self.build_swh_revision(
rev, commit, dir_id, revision_parents[rev])
swh_revision['id'] = git.compute_revision_sha1_git(
swh_revision)
self.log.debug('rev: %s, swhrev: %s, dir: %s' % (
rev,
hashutil.hash_to_hex(swh_revision['id']),
hashutil.hash_to_hex(dir_id)))
if (count % self.check_revision) == 0: # hash computation check
self.log.info('Checking hash computations on revision %s...' %
rev)
checked_dir_id = self.swh_revision_hash_tree_at_svn_revision(
rev)
if checked_dir_id != dir_id:
err = 'Hash tree computation divergence detected (%s != %s), stopping!' % ( # noqa
hashutil.hash_to_hex(dir_id),
hashutil.hash_to_hex(checked_dir_id)
)
raise ValueError(err)
if nextrev:
revision_parents[nextrev] = [swh_revision['id']]
yield swh_revision
def process_swh_revisions(self,
svnrepo,
revision_start,
revision_end,
revision_parents):
"""Process and store revision to swh (sent by blocks of
revision_packet_size)
Returns:
The latest revision stored.
"""
try:
swh_revision_gen = self.process_svn_revisions(svnrepo,
revision_start,
revision_end,
revision_parents)
revs = []
for revisions in utils.grouper(
swh_revision_gen,
self.config['revision_packet_size']):
revs = list(revisions)
self.log.info('Processed %s revisions: [%s, ...]' % (
len(revs), hashutil.hash_to_hex(revs[0]['id'])))
self.maybe_load_revisions(revs)
except Exception as e:
if revs:
# flush remaining revisions
self.maybe_load_revisions(revs)
# Take the last one as the last known revisions
known_swh_rev = revs[-1]
# Then notify something is wrong, and we stopped at that rev.
raise SvnLoaderEventful(e, swh_revision={
'id': known_swh_rev['id'],
})
raise e
return revs[-1]
def process_swh_occurrence(self, revision, origin_visit):
"""Process and load the occurrence pointing to the latest revision.
"""
occ = converters.build_swh_occurrence(
revision['id'], origin_visit['origin'], origin_visit['visit'])
self.log.debug('occ: %s' % occ)
self.maybe_load_occurrences([occ])
def prepare(self, *args, **kwargs):
destination_path = kwargs['destination_path']
# local svn url
svn_url = kwargs['svn_url']
origin_url = kwargs.get('origin_url')
self.visit_date = kwargs.get('visit_date')
origin = {
'url': origin_url if origin_url else svn_url,
'type': 'svn',
}
self.origin_id = self.send_origin(origin)
origin['id'] = self.origin_id
self.origin = origin
if 'swh_revision' in kwargs:
self.last_known_swh_revision = hashutil.hash_to_bytes(
kwargs['swh_revision'])
else:
self.last_known_swh_revision = None
self.svnrepo = self.get_svn_repo(svn_url, destination_path, origin)
self.fetch_history_id = self.open_fetch_history()
def get_origin(self):
"""Retrieve the origin we are working with.
"""
return self.origin # set in prepare method
def fetch_data(self):
"""We need to fetch and stream the data to store directly. So
fetch_data do actually nothing. The method ``store_data`` below is in
charge to do everything, fetch and store.
"""
pass
def store_data(self):
"""We need to fetch and stream the data to store directly because
there is too much data and state changes. Everything is
intertwined together (We receive patch and apply on disk and
compute at the hashes at the same time)
So every data to fetch and store is done here.
Note:
origin_visit and last_known_swh_revision must have been set in the
prepare method.
"""
origin_visit = {'origin': self.origin_id, 'visit': self.visit}
try:
latest_rev = self.process_repository(origin_visit,
self.last_known_swh_revision)
except SvnLoaderEventful as e:
self.log.error('Eventful partial visit. Detail: %s' % e)
latest_rev = e.swh_revision
self.process_swh_occurrence(latest_rev, origin_visit)
raise
except (SvnLoaderHistoryAltered, SvnLoaderUneventful) as e:
self.log.error('Uneventful visit. Detail: %s' % e)
raise
except Exception as e:
raise
else:
self.process_swh_occurrence(latest_rev, origin_visit)
class SWHSvnLoader(BaseSvnLoader):
"""Swh svn loader is the main implementation destined for production.
This implementation is able to deal with update on known svn repository.
Default policy:
Keep data as close as possible from the original svn data. We
only add information that are needed for update or continuing
from last known revision (svn revision and svn repository's
uuid).
"""
def __init__(self):
super().__init__()
def cleanup(self):
"""Clean after oneself.
This is in charge to flush the remaining data to write in swh storage.
And to clean up the svn repository's working representation on disk.
"""
self.svnrepo.clean_fs()
def swh_revision_hash_tree_at_svn_revision(self, revision):
"""Compute a given hash tree at specific revision.
"""
local_dirname, local_url = self.svnrepo.export_temporary(revision)
h = hashtree(local_url)['sha1_git']
self.svnrepo.clean_fs(local_dirname)
return h
def get_svn_repo(self, svn_url, destination_path, origin):
return svn.SWHSvnRepo(
svn_url, origin['id'], self.storage,
destination_path=destination_path)
def swh_previous_revision(self, prev_swh_revision=None):
"""Retrieve swh's previous revision if any.
"""
return self.svnrepo.swh_previous_revision(prev_swh_revision)
def check_history_not_altered(self, svnrepo, revision_start, swh_rev):
"""Given a svn repository, check if the history was not tampered with.
"""
revision_id = swh_rev['id']
parents = swh_rev['parents']
hash_data_per_revs = svnrepo.swh_hash_data_at_revision(revision_start)
rev = revision_start
rev, _, commit, objects_per_path = list(hash_data_per_revs)[0]
dir_id = objects_per_path[b'']['checksums']['sha1_git']
swh_revision = self.build_swh_revision(rev,
commit,
dir_id,
parents)
swh_revision_id = git.compute_revision_sha1_git(swh_revision)
return swh_revision_id == revision_id
def build_swh_revision(self, rev, commit, dir_id, parents):
"""Build the swh revision dictionary.
This adds:
- the `'synthetic`' flag to true
- the '`extra_headers`' containing the repository's uuid and the
svn revision number.
Args:
rev: the svn revision
commit: the commit metadata
dir_id: the upper tree's hash identifier
parents: the parents' identifiers
Returns:
The swh revision corresponding to the svn revision.
"""
return converters.build_swh_revision(rev,
commit,
self.svnrepo.uuid,
dir_id,
parents)
def init_from(self, partial_swh_revision, previous_swh_revision):
"""Function to determine from where to start from.
Args:
partial_swh_revision: A known revision from which
the previous loading did not finish.
known_previous_revision: A known revision from which the
previous loading did finish.
Returns:
The revision from which to start or None if nothing (fresh
start).
"""
if partial_swh_revision and not previous_swh_revision:
return partial_swh_revision
if not partial_swh_revision and previous_swh_revision:
return previous_swh_revision
if partial_swh_revision and previous_swh_revision:
# will determine from which to start from
extra_headers1 = dict(
partial_swh_revision['metadata']['extra_headers'])
extra_headers2 = dict(
previous_swh_revision['metadata']['extra_headers'])
rev_start1 = int(extra_headers1['svn_revision'])
rev_start2 = int(extra_headers2['svn_revision'])
if rev_start1 <= rev_start2:
return previous_swh_revision
return partial_swh_revision
return None
def process_repository(self, origin_visit, last_known_swh_revision=None):
svnrepo = self.svnrepo
# default configuration
revision_start = 1
revision_parents = {
revision_start: []
}
# Check if we already know a previous revision for that origin
swh_rev = self.swh_previous_revision()
# Determine from which known revision to start
swh_rev = self.init_from(last_known_swh_revision,
previous_swh_revision=swh_rev)
if swh_rev: # Yes, we do. Try and update it.
extra_headers = dict(swh_rev['metadata']['extra_headers'])
revision_start = int(extra_headers['svn_revision'])
revision_parents = {
revision_start: swh_rev['parents'],
}
self.log.debug('svn export --ignore-keywords %s@%s' % (
svnrepo.remote_url,
revision_start))
if swh_rev and not self.check_history_not_altered(
svnrepo,
revision_start,
swh_rev):
msg = 'History of svn %s@%s history modified. Skipping...' % (
svnrepo.remote_url, revision_start)
self.log.warn(msg)
raise SvnLoaderHistoryAltered
else:
# now we know history is ok, we start at next revision
revision_start = revision_start + 1
# and the parent become the latest know revision for
# that repository
revision_parents[revision_start] = [swh_rev['id']]
revision_end = svnrepo.head_revision()
self.log.info('[revision_start-revision_end]: [%s-%s]' % (
revision_start, revision_end))
if revision_start > revision_end and revision_start is not 1:
self.log.info('%s@%s already injected.' % (
svnrepo.remote_url, revision_end))
raise SvnLoaderUneventful
self.log.info('Processing %s.' % svnrepo)
# process and store revision to swh (sent by by blocks of
# 'revision_packet_size')
return self.process_swh_revisions(
svnrepo, revision_start, revision_end, revision_parents)
class SWHSvnLoaderFromDumpArchive(SWHSvnLoader):
"""Load a svn repository from an archive (containing a dump).
"""
def __init__(self, archive_path):
super().__init__()
self.log.info('Archive to mount and load %s' % archive_path)
self.temp_dir, self.repo_path = init_svn_repo_from_archive_dump(
archive_path)
def cleanup(self):
super().cleanup()
if self.temp_dir and os.path.exists(self.temp_dir):
self.log.debug('Clean up temp directory %s for project %s' % (
self.temp_dir, os.path.basename(self.repo_path)))
shutil.rmtree(self.temp_dir)
diff --git a/swh/loader/svn/ra.py b/swh/loader/svn/ra.py
index 7f624ee..2c0fb8b 100644
--- a/swh/loader/svn/ra.py
+++ b/swh/loader/svn/ra.py
@@ -1,507 +1,507 @@
# Copyright (C) 2016 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
"""Remote Access client to svn server.
"""
import click
import os
import shutil
import tempfile
from subvertpy import delta, properties
from subvertpy.ra import RemoteAccess, Auth, get_username_provider
from swh.model import git, hashutil
def compute_svn_link_metadata(linkpath, filetype, data):
"""Given a svn linkpath (raw file with format 'link <src-link>'),
compute the git metadata.
Args:
linkpath: absolute pathname of the svn link
filetype: the file's type according to svn, should only be 'link'
data: the link's content a.k.a the link's source file
Returns:
dict: a dictionary with the following keys:
- data: link's content
- length: link's content length
- name: basename of the link
- perms: git permission for link
- type: git type for link
- path: absolute path to the link on filesystem
Raises:
ValueError if the filetype does not match 'link'.
"""
if filetype != b'link':
raise ValueError(
'Do not deal with other type (%s) than link.' % (
linkpath, type))
link_metadata = hashutil.hash_data(data)
link_metadata.update({
'data': data,
'length': len(data),
'name': os.path.basename(linkpath),
'perms': git.GitPerm.LINK,
'type': git.GitType.BLOB,
'path': linkpath
})
return link_metadata
def apply_txdelta_handler(sbuf, target_stream):
"""Return a function that can be called repeatedly with txdelta windows.
When done, closes the target_stream.
Adapted from subvertpy.delta.apply_txdelta_handler to close the
stream when done.
Args:
sbuf: Source buffer
target_stream: Target stream to write to.
Returns:
Function to be called to apply txdelta windows
"""
def apply_window(window):
if window is None:
target_stream.close()
return # Last call
patch = delta.apply_txdelta_window(sbuf, window)
target_stream.write(patch)
return apply_window
class SWHFileEditor:
"""File Editor in charge of updating file on disk and memory objects.
"""
__slots__ = ['objects', 'path', 'fullpath', 'executable', 'link']
def __init__(self, objects, rootpath, path):
self.objects = objects
self.path = path
# default value: 0, 1: set the flag, 2: remove the exec flag
self.executable = 0
self.link = None
self.fullpath = os.path.join(rootpath, path)
def change_prop(self, key, value):
if key == properties.PROP_EXECUTABLE:
if value is None: # bit flip off
self.executable = 2
else:
self.executable = 1
elif key == properties.PROP_SPECIAL:
self.link = True
def __make_symlink(self):
"""Convert the svnlink to a symlink on disk.
This function expects self.fullpath to be a svn link.
Return:
tuple: The svnlink's data tuple:
- type (should be only 'link')
- <path-to-src>
"""
with open(self.fullpath, 'rb') as f:
filetype, src = f.read().split(b' ')
os.remove(self.fullpath)
os.symlink(src=src, dst=self.fullpath)
return filetype, src
def __make_svnlink(self):
"""Convert the symlink to a svnlink on disk.
Return:
The symlink's svnlink data (``b'type <path-to-src>'``)
"""
# we replace the symlink by a svnlink
src = os.readlink(self.fullpath)
os.remove(self.fullpath)
# to be able to patch the file
sbuf = b'link ' + src
with open(self.fullpath, 'wb') as f:
f.write(sbuf)
return sbuf
def apply_textdelta(self, base_checksum):
if os.path.lexists(self.fullpath):
if os.path.islink(self.fullpath):
sbuf = self.__make_svnlink()
self.link = True
else:
with open(self.fullpath, 'rb') as f:
sbuf = f.read()
else:
sbuf = b''
t = open(self.fullpath, 'wb')
return apply_txdelta_handler(sbuf, target_stream=t)
def close(self):
"""When done with the file, this is called.
So the file exists and is updated, we can:
- adapt accordingly its execution flag if any
- compute the objects' checksums
"""
if self.link:
filetype, source_link = self.__make_symlink()
self.objects[self.path] = {
'checksums': compute_svn_link_metadata(self.fullpath,
filetype=filetype,
data=source_link)
}
return
if self.executable == 1:
os.chmod(self.fullpath, 0o755)
elif self.executable == 2:
os.chmod(self.fullpath, 0o644)
# And now compute file's checksums
self.objects[self.path] = {
'checksums': git.compute_blob_metadata(self.fullpath)
}
def default_dictionary():
"""Default dictionary.
"""
return dict(checksums=None, children=set())
class BaseDirSWHEditor:
"""Base class implementation of dir editor.
- see :py:class:`SWHDirEditor` for an implementation that hashes every
+ see :class:`SWHDirEditor` for an implementation that hashes every
directory encountered.
- cf. :py:class:`SWHDirEditorNoEmptyFolder` for an implementation that
+ cf. :class:`SWHDirEditorNoEmptyFolder` for an implementation that
deletes empty folder
Instantiate a new class inheriting from this class and define the following
functions::
def update_checksum(self):
# Compute the checksums at current state
def open_directory(self, *args):
# Update an existing folder.
def add_directory(self, *args):
# Add a new one.
"""
__slots__ = ['objects', 'rootpath', 'path']
def __init__(self, objects, rootpath, path):
self.objects = objects
self.rootpath = rootpath
self.path = path
# build directory on init
os.makedirs(os.path.join(rootpath, path), exist_ok=True)
def add_child(self, path):
"""Add a children path to the actual objects for the current directory
seen as the parent.
Args:
path: The child to add
"""
d = self.objects.get(self.path, default_dictionary())
d['children'].add(path)
self.objects[self.path] = d
def remove_child(self, path):
"""Remove a path from the current objects.
The path can be resolved as link, file or directory.
This function takes also care of removing the link between the
child and the parent.
Args:
path: to remove from the current objects.
"""
entry_removed = self.objects.pop(path, None)
fpath = os.path.join(self.rootpath, path)
if entry_removed:
if 'children' in entry_removed: # dir
for child_path in entry_removed['children']:
self.remove_child(child_path)
parent = os.path.dirname(path)
if parent and parent in self.objects:
self.objects[parent]['children'].discard(path)
if os.path.lexists(fpath): # we want to catch broken symlink too
if os.path.isfile(fpath):
os.remove(fpath)
elif os.path.islink(fpath):
os.remove(fpath)
else:
shutil.rmtree(fpath)
def update_checksum(self):
raise NotImplementedError('This should be implemented.')
def open_directory(self, *args):
raise NotImplementedError('This should be implemented.')
def add_directory(self, *args):
raise NotImplementedError('This should be implemented.')
def open_file(self, *args):
"""Updating existing file.
"""
path = args[0].encode('utf-8')
self.add_child(path)
return SWHFileEditor(self.objects, rootpath=self.rootpath, path=path)
def add_file(self, path, copyfrom_path=None, copyfrom_rev=-1):
"""Creating a new file.
"""
path = path.encode('utf-8')
self.add_child(path)
return SWHFileEditor(self.objects, rootpath=self.rootpath, path=path)
def change_prop(self, key, value):
"""Change property callback on directory.
"""
if key == properties.PROP_EXTERNALS:
raise ValueError(
"Property '%s' detected. Not implemented yet." % key)
def delete_entry(self, path, revision):
"""Remove a path.
"""
self.remove_child(path.encode('utf-8'))
def close(self):
"""Function called when we finish walking a repository.
"""
self.update_checksum()
class SWHDirEditor(BaseDirSWHEditor):
"""Directory Editor in charge of updating directory hashes computation.
This implementation includes empty folder in the hash computation.
"""
def update_checksum(self):
"""Update the root path self.path's checksums according to the
children's objects.
This function is expected to be called when the folder has
been completely 'walked'.
"""
d = self.objects.get(self.path, default_dictionary())
# Retrieve the list of the current folder's children objects
ls_hashes = list(git.children_hashes(d['children'],
objects=self.objects))
d['checksums'] = git._compute_tree_metadata(self.path, ls_hashes)
self.objects[self.path] = d
def open_directory(self, *args):
"""Updating existing directory.
"""
path = args[0].encode('utf-8')
self.add_child(path)
return SWHDirEditor(self.objects, self.rootpath, path=path)
def add_directory(self, path, copyfrom_path=None, copyfrom_rev=-1):
"""Adding a new directory.
"""
path = path.encode('utf-8')
self.add_child(path)
return SWHDirEditor(self.objects, rootpath=self.rootpath, path=path)
class BaseSWHEditor:
"""SWH Base class editor in charge of receiving events.
"""
def __init__(self, rootpath, objects):
self.rootpath = rootpath
self.objects = objects
def set_target_revision(self, revnum):
pass
def abort(self):
pass
def close(self):
pass
def open_root(self, base_revnum):
raise NotImplementedError('Instantiate an swh dir editor of your '
' choice depending of the hash computation '
' policy you want')
class SWHEditor(BaseSWHEditor):
"""SWH Editor in charge of replaying svn events and computing objects
along.
This implementation accounts for empty folder during hash
computations.
"""
def open_root(self, base_revnum):
return SWHDirEditor(self.objects, rootpath=self.rootpath, path=b'')
class BaseSWHReplay:
"""Base replay class.
Their role is to compute objects for a particular revision.
This class is intended to be inherited to:
- initialize the editor (global loading policy depends on this editor)
- override the compute_hashes function in charge of computing
hashes between rev and rev+1
- cf. :py:class:`SWHReplayNoEmptyFolder` and :py:class:`SWHReplay` for
+ cf. :class:`SWHReplayNoEmptyFolder` and :class:`SWHReplay` for
instanciated classes.
"""
def replay(self, rev):
"""Replay svn actions between rev and rev+1.
This method updates in place the self.editor.objects's reference. This
also updates in place the filesystem.
Returns:
The updated objects
"""
self.conn.replay(rev, rev+1, self.editor)
return self.editor.objects
def compute_hashes(self, rev):
"""Compute hashes at revisions rev.
Expects the objects to be at previous revision's objects.
Args:
rev: The revision to start the replay from.
Returns:
The updated objects between rev and rev+1. Beware that this mutates
the filesystem at rootpath accordingly.
"""
raise NotImplementedError('This should be overridden by subclass')
class SWHReplay(BaseSWHReplay):
"""Replay class.
All folders and files are considered for hash computations.
"""
def __init__(self, conn, rootpath, objects=None):
self.conn = conn
self.rootpath = rootpath
self.editor = SWHEditor(rootpath=rootpath,
objects=objects if objects else {})
def compute_hashes(self, rev):
"""Compute hashes at revisions rev.
Expects the state to be at previous revision's objects.
Args:
rev: The revision to start the replay from.
Returns:
The updated objects between rev and rev+1. Beware that this
mutates the filesystem at rootpath accordingly.
"""
return self.replay(rev)
@click.command()
@click.option('--local-url', default='/tmp',
help="local svn working copy")
@click.option('--svn-url', default='file:///home/storage/svn/repos/pkg-fox',
help="svn repository's url.")
@click.option('--revision-start', default=1, type=click.INT,
help="svn repository's starting revision.")
@click.option('--revision-end', default=-1, type=click.INT,
help="svn repository's ending revision.")
@click.option('--debug/--nodebug', default=True,
help="Indicates if the server should run in debug mode.")
@click.option('--cleanup/--nocleanup', default=True,
help="Indicates whether to cleanup disk when done or not.")
def main(local_url, svn_url, revision_start, revision_end, debug, cleanup):
"""Script to present how to use SWHReplay class.
"""
conn = RemoteAccess(svn_url.encode('utf-8'),
auth=Auth([get_username_provider()]))
os.makedirs(local_url, exist_ok=True)
rootpath = tempfile.mkdtemp(prefix=local_url,
suffix='-'+os.path.basename(svn_url))
rootpath = rootpath.encode('utf-8')
# Do not go beyond the repository's latest revision
revision_end_max = conn.get_latest_revnum()
if revision_end == -1:
revision_end = revision_end_max
revision_end = min(revision_end, revision_end_max)
try:
replay = SWHReplay(conn, rootpath)
for rev in range(revision_start, revision_end+1):
objects = replay.compute_hashes(rev)
print('r%s %s' % (rev, hashutil.hash_to_hex(
objects[b'']['checksums']['sha1_git'])))
if debug:
print('%s' % rootpath.decode('utf-8'))
finally:
if cleanup:
if os.path.exists(rootpath):
shutil.rmtree(rootpath)
if __name__ == '__main__':
main()
diff --git a/swh/loader/svn/svn.py b/swh/loader/svn/svn.py
index a19515a..365b851 100644
--- a/swh/loader/svn/svn.py
+++ b/swh/loader/svn/svn.py
@@ -1,376 +1,376 @@
# Copyright (C) 2015-2016 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
"""SVN client in charge of iterating over svn logs and yield commit
representations including the hash tree/content computations per
svn commit.
"""
import os
import tempfile
import shutil
from subvertpy.ra import RemoteAccess, Auth, get_username_provider
from subvertpy import client, properties
from swh.model import git
from . import ra, utils, converters
# When log message contains empty data
DEFAULT_AUTHOR_MESSAGE = ''
class SvnRepoException(ValueError):
def __init__(self, svnrepo, e):
super().__init__(e)
self.svnrepo = svnrepo
class BaseSvnRepo():
"""Base svn repository representation for swh.
To override some of the behavior regarding the message log properties, you
can instantiate a subclass of this class and override::
def convert_commit_author(self, author)
def convert_commit_message(self, msg)
def convert_commit_date(self, date)
- see :py:class:`GitSvnSvnRepo`, :py:class:`SwhSvnRepo` for instanciation
+ see :class:`GitSvnSvnRepo`, :class:`SwhSvnRepo` for instanciation
example.
"""
def __init__(self, remote_url, origin_id, storage,
destination_path=None):
self.remote_url = remote_url.rstrip('/')
self.storage = storage
self.origin_id = origin_id
if destination_path:
os.makedirs(destination_path, exist_ok=True)
self.root_dir = destination_path
else:
self.root_dir = '/tmp'
auth = Auth([get_username_provider()])
# one connection for log iteration
self.conn_log = RemoteAccess(self.remote_url, auth=auth)
# another for replay
self.conn = RemoteAccess(self.remote_url, auth=auth)
# one client for update operation
self.client = client.Client(auth=auth)
self.local_dirname = tempfile.mkdtemp(
suffix='.swh.loader', prefix='tmp.', dir=self.root_dir)
local_name = os.path.basename(self.remote_url)
self.local_url = os.path.join(self.local_dirname, local_name).encode(
'utf-8')
self.uuid = self.conn.get_uuid().encode('utf-8')
def __str__(self):
return str({
'remote_url': self.remote_url,
'local_url': self.local_url,
'uuid': self.uuid,
'swh-origin': self.origin_id
})
def head_revision(self):
"""Retrieve current revision of the repository's working copy.
"""
return self.conn.get_latest_revnum()
def initial_revision(self):
"""Retrieve the initial revision from which the remote url appeared.
Note: This should always be 1 since we won't be dealing with in-depth
url.
"""
return 1
def convert_commit_message(self, msg):
"""Do something with message (e.g add extra line, etc...)
cf. SvnRepo for a simple implementation.
Args:
msg (str): the commit message to convert.
Returns:
The transformed message as bytes.
"""
raise NotImplementedError('Should be overridden by subclass.')
def convert_commit_date(self, date):
"""Convert the message date (e.g, convert into timestamp or whatever
makes sense to you.).
Args:
date (str): the commit date to convert.
Returns:
The transformed date.
"""
raise NotImplementedError('Should be overridden by subclass.')
def convert_commit_author(self, author):
"""Convert the commit author (e.g, convert into dict or whatever
makes sense to you.).
Args:
author (str): the commit author to convert.
Returns:
The transformed author as dict.
"""
raise NotImplementedError('Should be overridden by subclass.')
def __to_entry(self, log_entry):
changed_paths, rev, revprops, has_children = log_entry
author_date = self.convert_commit_date(
revprops.get(properties.PROP_REVISION_DATE))
author = self.convert_commit_author(
revprops.get(properties.PROP_REVISION_AUTHOR))
message = self.convert_commit_message(
revprops.get(properties.PROP_REVISION_LOG,
DEFAULT_AUTHOR_MESSAGE))
return {
'rev': rev,
'author_date': author_date,
'author_name': author,
'message': message,
}
def logs(self, revision_start, revision_end):
"""Stream svn logs between revision_start and revision_end by chunks of
block_size logs.
Yields revision and associated revision information between the
revision start and revision_end.
Args:
revision_start: the svn revision starting bound
revision_end: the svn revision ending bound
Yields:
tuple: tuple of revisions and logs:
- revisions: list of revisions in order
- logs: Dictionary with key revision number and value the log
entry. The log entry is a dictionary with the following keys:
- author_date: date of the commit
- author_name: name of the author
- message: commit message
"""
for log_entry in self.conn_log.iter_log(paths=None,
start=revision_start,
end=revision_end,
discover_changed_paths=False):
yield self.__to_entry(log_entry)
def export(self, revision):
"""Export the repository to a given version.
"""
self.client.export(self.remote_url,
to=self.local_url.decode('utf-8'),
rev=revision,
ignore_keywords=True)
def export_temporary(self, revision):
"""Export the repository to a given revision in a temporary location.
This is up to the caller of this function to clean up the
temporary location when done (cf. self.clean_fs method)
Args:
revision: Revision to export at
Returns:
The tuple local_dirname the temporary location root
folder, local_url where the repository was exported.
"""
local_dirname = tempfile.mkdtemp(
prefix='check-revision-%s.' % revision,
dir=self.root_dir)
local_name = os.path.basename(self.remote_url)
local_url = os.path.join(local_dirname, local_name)
self.client.export(
self.remote_url, to=local_url, rev=revision, ignore_keywords=True)
return local_dirname, local_url
def swh_previous_revision(self, previous_swh_revision=None):
"""Look for possible existing revision in swh.
Args:
previous_swh_revision: (optional) id of a possible
previous swh revision
Returns:
If previous_swh_revision is not None and do exists,
returns the complete instance.
Otherwise, check for a possible occurrence and returns the
targeted complete revision if it does exists.
Otherwise, returns None.
"""
storage = self.storage
# got no previous revision, will check if some occurrence
# already exists for that origin
if not previous_swh_revision:
occ = list(storage.occurrence_get(self.origin_id))
if occ:
revision_id = occ[0]['target']
revisions = list(storage.revision_get([revision_id]))
if revisions:
return revisions[0]
else:
revs = list(storage.revision_get([previous_swh_revision]))
if revs:
return revs[0]
def swh_hash_data_per_revision(self, start_revision, end_revision):
"""Compute swh hash data per each revision between start_revision and
end_revision.
Args:
start_revision: starting revision
end_revision: ending revision
Yields:
tuple (rev, nextrev, commit, objects_per_path)
- rev: current revision
- nextrev: next revision
- commit: commit data (author, date, message) for such revision
- objects_per_path: dictionary of path, swh hash data with type
"""
hashes = {}
for commit in self.logs(start_revision, end_revision):
rev = commit['rev']
hashes = self.swhreplay.compute_hashes(rev)
if rev == end_revision:
nextrev = None
else:
nextrev = rev + 1
yield rev, nextrev, commit, hashes
def swh_hash_data_at_revision(self, revision):
"""Compute the hash data at revision.
Expected to be used for update only.
"""
# Update the disk at revision
self.export(revision)
# Compute the current hashes on disk
hashes = git.compute_hashes_from_directory(self.local_url)
hashes = utils.convert_hashes_with_relative_path(
hashes,
rootpath=self.local_url)
# Update the replay collaborator with the right state
self.swhreplay = ra.SWHReplay(
conn=self.conn,
rootpath=self.local_url,
objects=hashes)
# Retrieve the commit information for revision
commit = list(self.logs(revision, revision))[0]
yield revision, revision + 1, commit, hashes
def clean_fs(self, local_dirname=None):
"""Clean up the local working copy.
Args:
local_dirname (str): Path to remove recursively if
provided. Otherwise, remove the temporary upper root tree
used for svn repository loading.
"""
if local_dirname:
shutil.rmtree(local_dirname)
else:
shutil.rmtree(self.local_dirname)
class SWHSvnRepo(BaseSvnRepo):
- """Same as :py:class:`BaseSvnRepo` except for:
+ """Same as :class:`BaseSvnRepo` except for:
- the commit message which is simply encoded
- the commit author is left as is.
- the commit timestamp is left as is.
"""
def __init__(self, remote_url, origin_id, storage,
destination_path=None):
super().__init__(remote_url, origin_id, storage,
destination_path=destination_path)
self.swhreplay = ra.SWHReplay(
conn=self.conn,
rootpath=self.local_url)
def convert_commit_message(self, msg):
"""Simply encode the commit message.
Args:
msg (str): the commit message to convert.
Returns:
The transformed message as bytes.
"""
return msg.encode('utf-8')
def convert_commit_date(self, date):
"""Convert the message commit date into a timestamp in swh format.
The precision is kept.
Args:
date (str): the commit date to convert.
Returns:
The transformed date.
"""
return converters.svn_date_to_swh_date(date)
def convert_commit_author(self, author):
"""Convert the commit author into an swh person.
The user becomes a dictionary of the form::
{
name: author,
email: '',
fullname: author
}
Args:
author (str): the commit author to convert.
Returns:
The transformed author as dict.
"""
return converters.svn_author_to_swh_person(author)

File Metadata

Mime Type
text/x-diff
Expires
Mon, Aug 18, 7:40 PM (1 w, 1 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3213586

Event Timeline

About · Developer information · Legal