Page MenuHomeSoftware Heritage
Files F9125051

No OneTemporary
Actions

View Options

diff --git a/PKG-INFO b/PKG-INFO
index 337a3aa..f6174fe 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,10 +1,10 @@
Metadata-Version: 1.0
Name: swh.loader.svn
-Version: 0.0.35
+Version: 0.0.36
Summary: Software Heritage Loader SVN
Home-page: https://forge.softwareheritage.org/diffusion/DLDSVN
Author: Software Heritage developers
Author-email: swh-devel@inria.fr
License: UNKNOWN
Description: UNKNOWN
Platform: UNKNOWN
diff --git a/swh.loader.svn.egg-info/PKG-INFO b/swh.loader.svn.egg-info/PKG-INFO
index 337a3aa..f6174fe 100644
--- a/swh.loader.svn.egg-info/PKG-INFO
+++ b/swh.loader.svn.egg-info/PKG-INFO
@@ -1,10 +1,10 @@
Metadata-Version: 1.0
Name: swh.loader.svn
-Version: 0.0.35
+Version: 0.0.36
Summary: Software Heritage Loader SVN
Home-page: https://forge.softwareheritage.org/diffusion/DLDSVN
Author: Software Heritage developers
Author-email: swh-devel@inria.fr
License: UNKNOWN
Description: UNKNOWN
Platform: UNKNOWN
diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py
index a069575..f6e3c89 100644
--- a/swh/loader/svn/loader.py
+++ b/swh/loader/svn/loader.py
@@ -1,544 +1,552 @@
# Copyright (C) 2015-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
"""Loader in charge of injecting either new or existing svn mirrors to
swh-storage.
"""
import os
import shutil
from swh.core import utils
from swh.model import hashutil
from swh.model.from_disk import Directory
from swh.model.identifiers import identifier_to_bytes, revision_identifier
from swh.model.identifiers import snapshot_identifier
from swh.loader.core.loader import SWHLoader
from . import svn, converters
from .utils import init_svn_repo_from_archive_dump
DEFAULT_BRANCH = b'master'
def _revision_id(revision):
return identifier_to_bytes(revision_identifier(revision))
def build_swh_snapshot(revision_id, branch=DEFAULT_BRANCH):
"""Build a swh snapshot from the revision id, origin id, and visit.
"""
return {
'id': None,
'branches': {
branch: {
'target': revision_id,
'target_type': 'revision',
}
}
}
class SvnLoaderEventful(ValueError):
"""A wrapper exception to transit the swh_revision onto which the
loading failed.
"""
def __init__(self, e, swh_revision):
super().__init__(e)
self.swh_revision = swh_revision
class SvnLoaderUneventful(ValueError):
"""'Loading did nothing' exception to transit the latest known
snapshot.
"""
def __init__(self, e):
super().__init__(e)
class SvnLoaderHistoryAltered(ValueError):
def __init__(self, e):
super().__init__(e)
class SWHSvnLoader(SWHLoader):
"""Swh svn loader to load an svn repository The repository is either
remote or local. The loader deals with update on an already
previously loaded repository.
Default policy:
Keep data as close as possible from the original svn data. We
only add information that are needed for update or continuing
from last known revision (svn revision and svn repository's
uuid).
"""
CONFIG_BASE_FILENAME = 'loader/svn'
ADDITIONAL_CONFIG = {
'check_revision': ('int', 1000),
+ 'debug': ('bool', False), # NOT FOR PRODUCTION, False is mandatory
}
def __init__(self):
super().__init__(logging_class='swh.loader.svn.SvnLoader')
self.check_revision = self.config['check_revision']
self.origin_id = None
+ self.debug = self.config['debug']
def cleanup(self):
"""Clean up the svn repository's working representation on disk.
"""
+ if self.debug:
+ self.log.error('''NOT FOR PRODUCTION - debug flag activated
+Local repository not cleaned up for investigation: %s''' % (
+ self.svnrepo.local_url.decode('utf-8'), ))
+ return
self.svnrepo.clean_fs()
def swh_revision_hash_tree_at_svn_revision(self, revision):
"""Compute and return the hash tree at a given svn revision.
Args:
rev (int): the svn revision we want to check
Returns:
The hash tree directory as bytes.
"""
local_dirname, local_url = self.svnrepo.export_temporary(revision)
h = Directory.from_disk(path=local_url).hash
self.svnrepo.clean_fs(local_dirname)
return h
def get_svn_repo(self, svn_url, destination_path, origin):
"""Instantiates the needed svnrepo collaborator to permit reading svn
repository.
Args:
svn_url (str): the svn repository url to read from
destination_path (str): the local path on disk to compute data
origin (int): the corresponding origin
Returns:
Instance of :mod:`swh.loader.svn.svn` clients
"""
return svn.SWHSvnRepo(
svn_url, origin['id'], self.storage,
destination_path=destination_path)
def swh_latest_snapshot_revision(self, origin_id,
previous_swh_revision=None):
"""Look for latest snapshot revision and returns it if any.
Args:
origin_id (int): Origin identifier
previous_swh_revision: (optional) id of a possible
previous swh revision
Returns:
dict: The latest known point in time. Dict with keys:
'revision': latest visited revision
'snapshot': latest snapshot
If None is found, return an empty dict.
"""
storage = self.storage
if not previous_swh_revision: # check latest snapshot's revision
latest_snap = storage.snapshot_get_latest(origin_id)
if latest_snap:
branches = latest_snap.get('branches')
if not branches:
return {}
branch = branches.get(DEFAULT_BRANCH)
if not branch:
return {}
target_type = branch['target_type']
if target_type != 'revision':
return {}
previous_swh_revision = branch['target']
else:
return {}
revs = list(storage.revision_get([previous_swh_revision]))
if revs:
return {
'snapshot': latest_snap,
'revision': revs[0]
}
return {}
def build_swh_revision(self, rev, commit, dir_id, parents):
"""Build the swh revision dictionary.
This adds:
- the `'synthetic`' flag to true
- the '`extra_headers`' containing the repository's uuid and the
svn revision number.
Args:
rev (dict): the svn revision
commit (dict): the commit metadata
dir_id (bytes): the upper tree's hash identifier
parents ([bytes]): the parents' identifiers
Returns:
The swh revision corresponding to the svn revision.
"""
return converters.build_swh_revision(rev,
commit,
self.svnrepo.uuid,
dir_id,
parents)
def check_history_not_altered(self, svnrepo, revision_start, swh_rev):
"""Given a svn repository, check if the history was not tampered with.
"""
revision_id = swh_rev['id']
parents = swh_rev['parents']
hash_data_per_revs = svnrepo.swh_hash_data_at_revision(revision_start)
rev = revision_start
rev, _, commit, _, root_dir = list(hash_data_per_revs)[0]
dir_id = root_dir.hash
swh_revision = self.build_swh_revision(rev,
commit,
dir_id,
parents)
swh_revision_id = _revision_id(swh_revision)
return swh_revision_id == revision_id
def process_repository(self, origin_visit,
last_known_swh_revision=None,
start_from_scratch=False):
"""The main idea of this function is to:
- iterate over the svn commit logs
- extract the svn commit log metadata
- compute the hashes from the current directory down to the file
- compute the equivalent swh revision
- send all those objects for storage
- create an swh occurrence pointing to the last swh revision seen
- send that occurrence for storage in swh-storage.
"""
svnrepo = self.svnrepo
- # default configuration
- revision_start = 1
+ revision_head = svnrepo.head_revision()
+ if revision_head == 0: # empty repository case
+ revision_start = 0
+ revision_end = 0
+ else: # default configuration
+ revision_start = svnrepo.initial_revision()
+ revision_end = revision_head
+
revision_parents = {
revision_start: []
}
if not start_from_scratch:
# Check if we already know a previous revision for that origin
if self.latest_snapshot:
swh_rev = self.latest_snapshot['revision']
else:
swh_rev = None
# Determine from which known revision to start
swh_rev = self.init_from(last_known_swh_revision,
previous_swh_revision=swh_rev)
if swh_rev: # Yes, we know a previous revision. Try and update it.
extra_headers = dict(swh_rev['metadata']['extra_headers'])
revision_start = int(extra_headers['svn_revision'])
revision_parents = {
revision_start: swh_rev['parents'],
}
self.log.debug('svn export --ignore-keywords %s@%s' % (
svnrepo.remote_url,
revision_start))
if swh_rev and not self.check_history_not_altered(
svnrepo,
revision_start,
swh_rev):
msg = 'History of svn %s@%s altered. ' \
'Skipping...' % (
svnrepo.remote_url, revision_start)
raise SvnLoaderHistoryAltered(msg)
# now we know history is ok, we start at next revision
revision_start = revision_start + 1
# and the parent become the latest know revision for
# that repository
revision_parents[revision_start] = [swh_rev['id']]
- revision_end = svnrepo.head_revision()
-
- self.log.info('[revision_start-revision_end]: [%s-%s]' % (
- revision_start, revision_end))
-
if revision_start > revision_end and revision_start is not 1:
msg = '%s@%s already injected.' % (svnrepo.remote_url,
revision_end)
raise SvnLoaderUneventful(msg)
- self.log.info('Processing %s.' % svnrepo)
+ self.log.info('Processing revisions [%s-%s] for %s' % (
+ revision_start, revision_end, svnrepo))
# process and store revision to swh (sent by by blocks of
# 'revision_packet_size')
return self.process_swh_revisions(
svnrepo, revision_start, revision_end, revision_parents)
def process_svn_revisions(self, svnrepo, revision_start, revision_end,
revision_parents):
"""Process revisions from revision_start to revision_end and send to
swh for storage.
At each svn revision, checkout the repository, compute the
tree hash and blobs and send for swh storage to store.
Then computes and yields the swh revision.
Note that at every self.check_revision, an svn export is done
and a hash tree is computed to check that no divergence
occurred.
Yields:
swh revision as a dictionary with keys, sha1_git, sha1, etc...
"""
gen_revs = svnrepo.swh_hash_data_per_revision(
revision_start,
revision_end)
swh_revision = None
count = 0
for rev, nextrev, commit, new_objects, root_directory in gen_revs:
count += 1
# Send the associated contents/directories
self.maybe_load_contents(new_objects.get('content', {}).values())
self.maybe_load_directories(
new_objects.get('directory', {}).values())
# compute the fs tree's checksums
dir_id = root_directory.hash
swh_revision = self.build_swh_revision(
rev, commit, dir_id, revision_parents[rev])
swh_revision['id'] = _revision_id(swh_revision)
self.log.debug('rev: %s, swhrev: %s, dir: %s' % (
rev,
hashutil.hash_to_hex(swh_revision['id']),
hashutil.hash_to_hex(dir_id)))
if (count % self.check_revision) == 0: # hash computation check
self.log.info('Checking hash computations on revision %s...' %
rev)
checked_dir_id = self.swh_revision_hash_tree_at_svn_revision(
rev)
if checked_dir_id != dir_id:
err = 'Hash tree computation divergence detected ' \
'(%s != %s), stopping!' % (
hashutil.hash_to_hex(dir_id),
hashutil.hash_to_hex(checked_dir_id))
raise ValueError(err)
if nextrev:
revision_parents[nextrev] = [swh_revision['id']]
yield swh_revision
def process_swh_revisions(self,
svnrepo,
revision_start,
revision_end,
revision_parents):
"""Process and store revision to swh (sent by blocks of
revision_packet_size)
Returns:
The latest revision stored.
"""
try:
swh_revision_gen = self.process_svn_revisions(svnrepo,
revision_start,
revision_end,
revision_parents)
revs = []
for revisions in utils.grouper(
swh_revision_gen,
self.config['revision_packet_size']):
revs = list(revisions)
-
- self.log.info('Processed %s revisions: [%s, ...]' % (
- len(revs), hashutil.hash_to_hex(revs[0]['id'])))
self.maybe_load_revisions(revs)
+ self.log.debug('Processed %s revisions: [%s, ...]' % (
+ len(revs), hashutil.hash_to_hex(revs[0]['id'])))
except Exception as e:
if revs:
# flush remaining revisions
self.maybe_load_revisions(revs)
# Take the last one as the last known revisions
known_swh_rev = revs[-1]
_id = known_swh_rev.get('id')
if not _id:
_id = _revision_id(known_swh_rev)
# Then notify something is wrong, and we stopped at that rev.
raise SvnLoaderEventful(e, swh_revision={
'id': _id,
})
raise e
return revs[-1]
def process_swh_snapshot(self, revision=None, snapshot=None):
"""Create the snapshot either from existing snapshot or revision.
"""
if snapshot:
snap = snapshot
elif revision:
snap = build_swh_snapshot(revision['id'])
snap['id'] = identifier_to_bytes(snapshot_identifier(snap))
else:
return None
self.log.debug('snapshot: %s' % snap)
self.maybe_load_snapshot(snap)
def prepare(self, *args, **kwargs):
self.args = args
destination_path = kwargs['destination_path']
# local svn url
svn_url = kwargs['svn_url']
origin_url = kwargs.get('origin_url')
self.visit_date = kwargs.get('visit_date')
self.start_from_scratch = kwargs.get('start_from_scratch', False)
origin = {
'url': origin_url if origin_url else svn_url,
'type': 'svn',
}
self.origin_id = self.send_origin(origin)
origin['id'] = self.origin_id
self.origin = origin
if 'swh_revision' in kwargs:
self.last_known_swh_revision = hashutil.hash_to_bytes(
kwargs['swh_revision'])
else:
self.last_known_swh_revision = None
self.latest_snapshot = self.swh_latest_snapshot_revision(
self.origin_id, self.last_known_swh_revision)
self.svnrepo = self.get_svn_repo(svn_url, destination_path, origin)
def get_origin(self):
"""Retrieve the origin we are working with (setup-ed in the prepare
method)
"""
return self.origin # set in prepare method
def fetch_data(self):
"""We need to fetch and stream the data to store directly. So
fetch_data do actually nothing. The method ``store_data`` below is in
charge to do everything, fetch and store.
"""
pass
def store_data(self):
"""We need to fetch and stream the data to store directly because
there is too much data and state changes. Everything is
intertwined together (We receive patch and apply on disk and
compute at the hashes at the same time)
So every data to fetch and store is done here.
Note:
origin_visit and last_known_swh_revision must have been set in the
prepare method.
"""
origin_visit = {'origin': self.origin_id, 'visit': self.visit}
try:
latest_rev = self.process_repository(
origin_visit,
last_known_swh_revision=self.last_known_swh_revision,
start_from_scratch=self.start_from_scratch)
except SvnLoaderUneventful as e: # uneventful visit
self.log.info('Uneventful visit. Detail: %s' % e)
if self.latest_snapshot and 'snapshot' in self.latest_snapshot:
snapshot = self.latest_snapshot['snapshot']
self.process_swh_snapshot(snapshot=snapshot)
except SvnLoaderEventful as e:
self.log.error('Eventful partial visit. Detail: %s' % e)
latest_rev = e.swh_revision
self.process_swh_snapshot(revision=latest_rev)
raise
except Exception as e:
if self.latest_snapshot and 'snapshot' in self.latest_snapshot:
snapshot = self.latest_snapshot['snapshot']
self.process_swh_snapshot(snapshot=snapshot)
raise
else:
self.process_swh_snapshot(revision=latest_rev)
def init_from(self, partial_swh_revision, previous_swh_revision):
"""Function to determine from where to start from.
Args:
partial_swh_revision: A known revision from which
the previous loading did not finish.
known_previous_revision: A known revision from which the
previous loading did finish.
Returns:
The revision from which to start or None if nothing (fresh
start).
"""
if partial_swh_revision and not previous_swh_revision:
return partial_swh_revision
if not partial_swh_revision and previous_swh_revision:
return previous_swh_revision
if partial_swh_revision and previous_swh_revision:
# will determine from which to start from
extra_headers1 = dict(
partial_swh_revision['metadata']['extra_headers'])
extra_headers2 = dict(
previous_swh_revision['metadata']['extra_headers'])
rev_start1 = int(extra_headers1['svn_revision'])
rev_start2 = int(extra_headers2['svn_revision'])
if rev_start1 <= rev_start2:
return previous_swh_revision
return partial_swh_revision
return None
class SWHSvnLoaderFromDumpArchive(SWHSvnLoader):
"""Uncompress an archive containing an svn dump, mount the svn dump as
an svn repository and load said repository.
"""
def __init__(self, archive_path):
super().__init__()
self.log.info('Archive to mount and load %s' % archive_path)
self.temp_dir, self.repo_path = init_svn_repo_from_archive_dump(
archive_path)
def cleanup(self):
super().cleanup()
if self.temp_dir and os.path.exists(self.temp_dir):
self.log.debug('Clean up temp directory %s for project %s' % (
self.temp_dir, os.path.basename(self.repo_path)))
shutil.rmtree(self.temp_dir)
diff --git a/swh/loader/svn/ra.py b/swh/loader/svn/ra.py
index 866614e..033b8d0 100644
--- a/swh/loader/svn/ra.py
+++ b/swh/loader/svn/ra.py
@@ -1,433 +1,441 @@
-# Copyright (C) 2016 The Software Heritage developers
+# Copyright (C) 2016-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
"""Remote Access client to svn server.
"""
import click
import os
import shutil
import tempfile
from subvertpy import delta, properties
from subvertpy.ra import RemoteAccess, Auth, get_username_provider
from swh.model import hashutil
from swh.model.from_disk import Content, Directory
def apply_txdelta_handler(sbuf, target_stream):
"""Return a function that can be called repeatedly with txdelta windows.
When done, closes the target_stream.
Adapted from subvertpy.delta.apply_txdelta_handler to close the
stream when done.
Args:
sbuf: Source buffer
target_stream: Target stream to write to.
Returns:
Function to be called to apply txdelta windows
"""
def apply_window(window, sbuf=sbuf, target_stream=target_stream):
if window is None:
target_stream.close()
return # Last call
patch = delta.apply_txdelta_window(sbuf, window)
target_stream.write(patch)
return apply_window
def read_svn_link(data):
"""Read the svn link's content.
Args:
data (bytes): svn link's raw content
Returns:
The tuple of (filetype, destination path)
"""
split_byte = b' '
filetype, *src = data.split(split_byte)
src = split_byte.join(src)
return filetype, src
def is_file_an_svnlink_p(fullpath):
"""Determine if a filepath is an svnlink or something else.
Args:
fullpath (str/bytes): Full path to the potential symlink to check
Return:
boolean value to determine if it's indeed a symlink (as per
svn) or not.
"""
with open(fullpath, 'rb') as f:
filetype, src = read_svn_link(f.read())
return filetype == b'link', src
+DEFAULT_FLAG = 0
+EXEC_FLAG = 1
+NOEXEC_FLAG = 2
+
+
class SWHFileEditor:
"""File Editor in charge of updating file on disk and memory objects.
"""
__slots__ = ['directory', 'path', 'fullpath', 'executable', 'link']
def __init__(self, directory, rootpath, path):
self.directory = directory
self.path = path
# default value: 0, 1: set the flag, 2: remove the exec flag
- self.executable = 0
+ self.executable = DEFAULT_FLAG
self.link = None
self.fullpath = os.path.join(rootpath, path)
def change_prop(self, key, value):
if key == properties.PROP_EXECUTABLE:
if value is None: # bit flip off
- self.executable = 2
+ self.executable = NOEXEC_FLAG
else:
- self.executable = 1
+ self.executable = EXEC_FLAG
elif key == properties.PROP_SPECIAL:
# Possibly a symbolic link. We cannot check further at
- # that moment though since patch(s) are not applied yet
+ # that moment though, patch(s) not being applied yet
self.link = True
def __make_symlink(self, src):
"""Convert the svnlink to a symlink on disk.
This function expects self.fullpath to be a svn link.
Args:
src (bytes): Path to the link's source
Return:
tuple: The svnlink's data tuple:
- type (should be only 'link')
- <path-to-src>
"""
os.remove(self.fullpath)
os.symlink(src=src, dst=self.fullpath)
def __make_svnlink(self):
"""Convert the symlink to a svnlink on disk.
Return:
The symlink's svnlink data (``b'type <path-to-src>'``)
"""
# we replace the symlink by a svnlink
# to be able to patch the file on future commits
src = os.readlink(self.fullpath)
os.remove(self.fullpath)
sbuf = b'link ' + src
with open(self.fullpath, 'wb') as f:
f.write(sbuf)
return sbuf
def apply_textdelta(self, base_checksum):
if os.path.lexists(self.fullpath):
if os.path.islink(self.fullpath):
# svn does not deal with symlink so we tranform into
# real svn symlink for potential patching in later
# commits
sbuf = self.__make_svnlink()
self.link = True
else:
with open(self.fullpath, 'rb') as f:
sbuf = f.read()
else:
sbuf = b''
t = open(self.fullpath, 'wb')
return apply_txdelta_handler(sbuf, target_stream=t)
def close(self):
"""When done with the file, this is called.
So the file exists and is updated, we can:
- adapt accordingly its execution flag if any
- compute the objects' checksums
- replace the svnlink with a real symlink (for disk
computation purposes)
"""
+ is_link = None
+
if self.link:
# can only check now that the link is a real one
# since patch has been applied
is_link, src = is_file_an_svnlink_p(self.fullpath)
if is_link:
self.__make_symlink(src)
else: # not a real link...
self.link = False
- if self.executable == 1:
- os.chmod(self.fullpath, 0o755)
- elif self.executable == 2:
- os.chmod(self.fullpath, 0o644)
+ if not is_link: # if a link, do nothing regarding flag
+ if self.executable == EXEC_FLAG:
+ os.chmod(self.fullpath, 0o755)
+ elif self.executable == NOEXEC_FLAG:
+ os.chmod(self.fullpath, 0o644)
# And now compute file's checksums
self.directory[self.path] = Content.from_file(path=self.fullpath,
data=True)
class BaseDirSWHEditor:
"""Base class implementation of dir editor.
see :class:`SWHDirEditor` for an implementation that hashes every
directory encountered.
Instantiate a new class inheriting from this class and define the following
functions::
def update_checksum(self):
# Compute the checksums at current state
def open_directory(self, *args):
# Update an existing folder.
def add_directory(self, *args):
# Add a new one.
"""
__slots__ = ['directory', 'rootpath']
def __init__(self, directory, rootpath):
self.directory = directory
self.rootpath = rootpath
# build directory on init
os.makedirs(rootpath, exist_ok=True)
def remove_child(self, path):
"""Remove a path from the current objects.
The path can be resolved as link, file or directory.
This function takes also care of removing the link between the
child and the parent.
Args:
path: to remove from the current objects.
"""
try:
entry_removed = self.directory[path]
except KeyError:
entry_removed = None
else:
del self.directory[path]
fpath = os.path.join(self.rootpath, path)
if isinstance(entry_removed, Directory):
shutil.rmtree(fpath)
else:
os.remove(fpath)
def update_checksum(self):
raise NotImplementedError('This should be implemented.')
def open_directory(self, *args):
raise NotImplementedError('This should be implemented.')
def add_directory(self, *args):
raise NotImplementedError('This should be implemented.')
def open_file(self, *args):
"""Updating existing file.
"""
path = os.fsencode(args[0])
self.directory[path] = Content()
return SWHFileEditor(self.directory, rootpath=self.rootpath, path=path)
def add_file(self, path, copyfrom_path=None, copyfrom_rev=-1):
"""Creating a new file.
"""
path = os.fsencode(path)
self.directory[path] = Content()
return SWHFileEditor(self.directory, self.rootpath, path)
def change_prop(self, key, value):
"""Change property callback on directory.
"""
if key == properties.PROP_EXTERNALS:
raise ValueError(
"Property '%s' detected. Not implemented yet." % key)
def delete_entry(self, path, revision):
"""Remove a path.
"""
self.remove_child(path.encode('utf-8'))
def close(self):
"""Function called when we finish walking a repository.
"""
self.update_checksum()
class SWHDirEditor(BaseDirSWHEditor):
"""Directory Editor in charge of updating directory hashes computation.
This implementation includes empty folder in the hash computation.
"""
def update_checksum(self):
"""Update the root path self.path's checksums according to the
children's objects.
This function is expected to be called when the folder has
been completely 'walked'.
"""
pass
def open_directory(self, *args):
"""Updating existing directory.
"""
return self
def add_directory(self, path, copyfrom_path=None, copyfrom_rev=-1):
"""Adding a new directory.
"""
path = os.fsencode(path)
os.makedirs(os.path.join(self.rootpath, path), exist_ok=True)
self.directory[path] = Directory()
return self
class SWHEditor:
"""SWH Editor in charge of replaying svn events and computing objects
along.
This implementation accounts for empty folder during hash
computations.
"""
def __init__(self, rootpath, directory):
self.rootpath = rootpath
self.directory = directory
def set_target_revision(self, revnum):
pass
def abort(self):
pass
def close(self):
pass
def open_root(self, base_revnum):
return SWHDirEditor(self.directory, rootpath=self.rootpath)
class SWHReplay:
"""Replay class.
"""
def __init__(self, conn, rootpath, directory=None):
self.conn = conn
self.rootpath = rootpath
if directory is None:
directory = Directory()
self.directory = directory
self.editor = SWHEditor(rootpath=rootpath, directory=directory)
def replay(self, rev):
"""Replay svn actions between rev and rev+1.
This method updates in place the self.editor.directory, as well as the
filesystem.
Returns:
The updated root directory
"""
self.conn.replay(rev, rev+1, self.editor)
return self.editor.directory
def compute_hashes(self, rev):
"""Compute hashes at revisions rev.
Expects the state to be at previous revision's objects.
Args:
rev: The revision to start the replay from.
Returns:
The updated objects between rev and rev+1. Beware that this
mutates the filesystem at rootpath accordingly.
"""
self.replay(rev)
return self.directory.collect()
@click.command()
@click.option('--local-url', default='/tmp',
help="local svn working copy")
@click.option('--svn-url', default='file:///home/storage/svn/repos/pkg-fox',
help="svn repository's url.")
@click.option('--revision-start', default=1, type=click.INT,
help="svn repository's starting revision.")
@click.option('--revision-end', default=-1, type=click.INT,
help="svn repository's ending revision.")
@click.option('--debug/--nodebug', default=True,
help="Indicates if the server should run in debug mode.")
@click.option('--cleanup/--nocleanup', default=True,
help="Indicates whether to cleanup disk when done or not.")
def main(local_url, svn_url, revision_start, revision_end, debug, cleanup):
"""Script to present how to use SWHReplay class.
"""
conn = RemoteAccess(svn_url.encode('utf-8'),
auth=Auth([get_username_provider()]))
os.makedirs(local_url, exist_ok=True)
rootpath = tempfile.mkdtemp(prefix=local_url,
suffix='-'+os.path.basename(svn_url))
rootpath = os.fsencode(rootpath)
# Do not go beyond the repository's latest revision
revision_end_max = conn.get_latest_revnum()
if revision_end == -1:
revision_end = revision_end_max
revision_end = min(revision_end, revision_end_max)
try:
replay = SWHReplay(conn, rootpath)
for rev in range(revision_start, revision_end+1):
objects = replay.compute_hashes(rev)
print("r%s %s (%s new contents, %s new directories)" % (
rev,
hashutil.hash_to_hex(replay.directory.hash),
len(objects.get('content', {})),
len(objects.get('directory', {})),
))
if debug:
print('%s' % rootpath.decode('utf-8'))
finally:
if cleanup:
if os.path.exists(rootpath):
shutil.rmtree(rootpath)
if __name__ == '__main__':
main()
diff --git a/swh/loader/svn/svn.py b/swh/loader/svn/svn.py
index 57ea158..0682c3d 100644
--- a/swh/loader/svn/svn.py
+++ b/swh/loader/svn/svn.py
@@ -1,344 +1,276 @@
# Copyright (C) 2015-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
"""SVN client in charge of iterating over svn logs and yield commit
-representations including the hash tree/content computations per
-svn commit.
+representations including the hash tree/content computations per svn
+commit.
"""
import os
import tempfile
import shutil
from subvertpy.ra import RemoteAccess, Auth, get_username_provider
from subvertpy import client, properties
from swh.model.from_disk import Directory
from . import ra, converters
# When log message contains empty data
DEFAULT_AUTHOR_MESSAGE = ''
class SvnRepoException(ValueError):
def __init__(self, svnrepo, e):
super().__init__(e)
self.svnrepo = svnrepo
-class BaseSvnRepo():
- """Base svn repository representation for swh.
-
- To override some of the behavior regarding the message log properties, you
- can instantiate a subclass of this class and override::
-
- def convert_commit_author(self, author)
- def convert_commit_message(self, msg)
- def convert_commit_date(self, date)
-
- see :class:`GitSvnSvnRepo`, :class:`SwhSvnRepo` for instanciation
- example.
+class SWHSvnRepo:
+ """SWH's svn repository representation.
"""
def __init__(self, remote_url, origin_id, storage,
destination_path=None):
self.remote_url = remote_url.rstrip('/')
self.storage = storage
self.origin_id = origin_id
if destination_path:
os.makedirs(destination_path, exist_ok=True)
self.root_dir = destination_path
else:
self.root_dir = '/tmp'
auth = Auth([get_username_provider()])
# one connection for log iteration
self.conn_log = RemoteAccess(self.remote_url, auth=auth)
# another for replay
self.conn = RemoteAccess(self.remote_url, auth=auth)
# one client for update operation
self.client = client.Client(auth=auth)
self.local_dirname = tempfile.mkdtemp(suffix='.tmp',
prefix='swh.loader.svn.',
dir=self.root_dir)
local_name = os.path.basename(self.remote_url)
self.local_url = os.path.join(self.local_dirname, local_name).encode(
'utf-8')
self.uuid = self.conn.get_uuid().encode('utf-8')
+ self.swhreplay = ra.SWHReplay(conn=self.conn, rootpath=self.local_url)
def __str__(self):
return str({
+ 'swh-origin': self.origin_id,
'remote_url': self.remote_url,
'local_url': self.local_url,
'uuid': self.uuid,
- 'swh-origin': self.origin_id
})
def head_revision(self):
- """Retrieve current revision of the repository's working copy.
+ """Retrieve current head revision.
"""
return self.conn.get_latest_revnum()
def initial_revision(self):
"""Retrieve the initial revision from which the remote url appeared.
- Note: This should always be 1 since we won't be dealing with in-depth
- url.
"""
return 1
def convert_commit_message(self, msg):
- """Do something with message (e.g add extra line, etc...)
-
- cf. SvnRepo for a simple implementation.
+ """Simply encode the commit message.
Args:
msg (str): the commit message to convert.
Returns:
The transformed message as bytes.
"""
- raise NotImplementedError('Should be overridden by subclass.')
+ if isinstance(msg, bytes):
+ return msg
+ return msg.encode('utf-8')
def convert_commit_date(self, date):
- """Convert the message date (e.g, convert into timestamp or whatever
- makes sense to you.).
+ """Convert the message commit date into a timestamp in swh format.
+ The precision is kept.
- Args:
- date (str): the commit date to convert.
+ Args:
+ date (str): the commit date to convert.
- Returns:
- The transformed date.
+ Returns:
+ The transformed date.
"""
- raise NotImplementedError('Should be overridden by subclass.')
+ return converters.svn_date_to_swh_date(date)
def convert_commit_author(self, author):
- """Convert the commit author (e.g, convert into dict or whatever
- makes sense to you.).
+ """Convert the commit author into an swh person.
+
+ The user becomes a dictionary of the form::
+
+ {
+ name: author,
+ email: '',
+ fullname: author
+ }
Args:
author (str): the commit author to convert.
Returns:
The transformed author as dict.
"""
- raise NotImplementedError('Should be overridden by subclass.')
+ return converters.svn_author_to_swh_person(author)
def __to_entry(self, log_entry):
changed_paths, rev, revprops, has_children = log_entry
author_date = self.convert_commit_date(
revprops.get(properties.PROP_REVISION_DATE))
author = self.convert_commit_author(
revprops.get(properties.PROP_REVISION_AUTHOR))
message = self.convert_commit_message(
revprops.get(properties.PROP_REVISION_LOG,
DEFAULT_AUTHOR_MESSAGE))
return {
'rev': rev,
'author_date': author_date,
'author_name': author,
'message': message,
}
def logs(self, revision_start, revision_end):
"""Stream svn logs between revision_start and revision_end by chunks of
block_size logs.
Yields revision and associated revision information between the
revision start and revision_end.
Args:
revision_start: the svn revision starting bound
revision_end: the svn revision ending bound
Yields:
tuple: tuple of revisions and logs:
- revisions: list of revisions in order
- logs: Dictionary with key revision number and value the log
entry. The log entry is a dictionary with the following keys:
- author_date: date of the commit
- author_name: name of the author
- message: commit message
"""
for log_entry in self.conn_log.iter_log(paths=None,
start=revision_start,
end=revision_end,
discover_changed_paths=False):
yield self.__to_entry(log_entry)
def export(self, revision):
"""Export the repository to a given version.
"""
self.client.export(self.remote_url,
to=self.local_url.decode('utf-8'),
rev=revision,
ignore_keywords=True)
def export_temporary(self, revision):
"""Export the repository to a given revision in a temporary location.
This is up to the caller of this function to clean up the
temporary location when done (cf. self.clean_fs method)
Args:
revision: Revision to export at
Returns:
The tuple local_dirname the temporary location root
folder, local_url where the repository was exported.
"""
local_dirname = tempfile.mkdtemp(
prefix='check-revision-%s.' % revision,
dir=self.local_dirname)
local_name = os.path.basename(self.remote_url)
local_url = os.path.join(local_dirname, local_name)
self.client.export(
self.remote_url, to=local_url, rev=revision, ignore_keywords=True)
return local_dirname, os.fsencode(local_url)
def swh_hash_data_per_revision(self, start_revision, end_revision):
"""Compute swh hash data per each revision between start_revision and
end_revision.
Args:
start_revision: starting revision
end_revision: ending revision
Yields:
tuple (rev, nextrev, commit, objects_per_path)
- rev: current revision
- nextrev: next revision
- commit: commit data (author, date, message) for such revision
- objects_per_path: dictionary of path, swh hash data with type
"""
for commit in self.logs(start_revision, end_revision):
rev = commit['rev']
objects = self.swhreplay.compute_hashes(rev)
if rev == end_revision:
nextrev = None
else:
nextrev = rev + 1
yield rev, nextrev, commit, objects, self.swhreplay.directory
def swh_hash_data_at_revision(self, revision):
"""Compute the hash data at revision.
Expected to be used for update only.
"""
# Update the disk at revision
self.export(revision)
# Compute the current hashes on disk
directory = Directory.from_disk(path=os.fsencode(self.local_url),
save_path=True)
# Update the replay collaborator with the right state
self.swhreplay = ra.SWHReplay(
conn=self.conn,
rootpath=self.local_url,
directory=directory)
# Retrieve the commit information for revision
commit = list(self.logs(revision, revision))[0]
yield revision, revision + 1, commit, {}, directory
def clean_fs(self, local_dirname=None):
"""Clean up the local working copy.
Args:
local_dirname (str): Path to remove recursively if
provided. Otherwise, remove the temporary upper root tree
used for svn repository loading.
"""
if local_dirname:
shutil.rmtree(local_dirname)
else:
shutil.rmtree(self.local_dirname)
-
-
-class SWHSvnRepo(BaseSvnRepo):
- """Same as :class:`BaseSvnRepo` except for:
-
- - the commit message which is simply encoded
- - the commit author is left as is.
- - the commit timestamp is left as is.
-
- """
- def __init__(self, remote_url, origin_id, storage,
- destination_path=None):
- super().__init__(remote_url, origin_id, storage,
- destination_path=destination_path)
- self.swhreplay = ra.SWHReplay(
- conn=self.conn,
- rootpath=self.local_url)
-
- def convert_commit_message(self, msg):
- """Simply encode the commit message.
-
- Args:
- msg (str): the commit message to convert.
-
- Returns:
- The transformed message as bytes.
-
- """
- if isinstance(msg, bytes):
- return msg
- return msg.encode('utf-8')
-
- def convert_commit_date(self, date):
- """Convert the message commit date into a timestamp in swh format.
- The precision is kept.
-
- Args:
- date (str): the commit date to convert.
-
- Returns:
- The transformed date.
-
- """
- return converters.svn_date_to_swh_date(date)
-
- def convert_commit_author(self, author):
- """Convert the commit author into an swh person.
-
- The user becomes a dictionary of the form::
-
- {
- name: author,
- email: '',
- fullname: author
- }
-
- Args:
- author (str): the commit author to convert.
-
- Returns:
- The transformed author as dict.
-
- """
- return converters.svn_author_to_swh_person(author)
diff --git a/version.txt b/version.txt
index 093e91f..d1671fc 100644
--- a/version.txt
+++ b/version.txt
@@ -1 +1 @@
-v0.0.35-0-ga892873
\ No newline at end of file
+v0.0.36-0-g1c42b93
\ No newline at end of file

File Metadata

Mime Type
text/x-diff
Expires
Sat, Jun 21, 7:58 PM (3 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3365633

Event Timeline

About · Developer information · Legal