diff --git a/README b/README index 97c2e23..07e3fe3 100644 --- a/README +++ b/README @@ -1,118 +1,124 @@ swh-loader-svn ============== Documents are in the ./docs folder: - Specification: ./docs/swh-loader-svn.txt - Comparison performance with git-svn: ./docs/comparison-git-svn-swh-svn.org # Configuration file ## Location Either: - /etc/softwareheritage/loader/svn.ini - ~/.config/swh/loader/svn.ini - ~/.swh/loader/svn.ini ## Default options ``` [main] storage_class = remote_storage storage_args = http://localhost:5000/ send_contents = True send_directories = True send_revisions = True send_releases = True send_occurrences = True # nb of max contents to send for storage content_packet_size = 100 # 100 Mib of content data content_packet_block_size_bytes = 104857600 # limit for swh content storage for one blob (beyond that limit, the # content's data is not sent for storage) content_packet_size_bytes = 1073741824 directory_packet_size = 2500 revision_packet_size = 1000 release_packet_size = 1 occurrence_packet_size = 1 # Flags -with_svn_update = True with_policy = swh ``` +This policy does not: +- alter anything in regards of svn data +- adds an extra-headers to permit to check and update from last known point + ## git-svn like For a git-svn like (ignore empty folder, no extra metadata, adapt -user's name with repository's uuid, add extra commit line), adapt to -the previous options with the following: +user's name with repository's uuid, add extra commit line, no update +policy), adapt to the previous options with the following: ```txt -with_svn_update = False with_policy = gitsvn ``` -Note: The 'gitsvn' policy automatically deactivate the svn update -behavior (we need the extra metadata added with the swh policy for -this to work so...) +This policy does alter things: + +- it adds the repo-uuid on author's name (author@) -> thus + in effects, altering the git hash computation of revisions +- truncates date time commits so some precision is lost on date +- removes empty folder if any are checkouted on disk + # Starting one instance ## worker's configuration file The file is either at: - /etc/softwareheritage/worker.ini - ~/.config/swh/worker.ini - ~/.swh/worker.ini ## configuration content With at least the following module (swh.loader.svn.tasks) and queue (swh_loader_svn): ``` [main] task_broker = amqp://guest@localhost// task_modules = swh.loader.svn.tasks,swh.loader.tar.tasks task_queues = swh_loader_svn, swh_loader_tar task_soft_time_limit = 0 ``` swh.loader.svn.tasks and swh_loader_svn are the important entries here. ## start worker instance To start a current worker instance: ```sh python3 -m celery worker --app=swh.scheduler.celery_backend.config.app \ --pool=prefork \ --concurrency=10 \ -Ofair \ --loglevel=debug 2>&1 ``` ## Produce a repository to load Either one repository: ```sh python3 -u -m swh.loader.svn.producer --svn-url file:///home/storage/svn/repos/pkg-fox ``` or a bunch: ```sh cat ~/svn-repository-list | python3 -m swh.loader.svn.producer ``` svn-repository is a list of svn repository urls (one per line). Something like: ```txt svn://svn.debian.org/svn/pkg-fox/ svn://svn.debian.org/svn/glibc-bsd/ svn://svn.debian.org/svn/pkg-voip/ svn://svn.debian.org/svn/python-modules/ svn://svn.debian.org/svn/pkg-gnome/ ``` diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py index a36a0c0..73647f4 100644 --- a/swh/loader/svn/loader.py +++ b/swh/loader/svn/loader.py @@ -1,315 +1,379 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Loader in charge of injecting either new or existing svn mirrors to swh-storage. """ import datetime from swh.core import utils from swh.model import git, hashutil from swh.model.git import GitType from swh.loader.core.loader import SWHLoader from . import svn, converters class BaseSvnLoader(SWHLoader): """Base Svn loader to load one svn repository. There exists 2 different policies: - - git-svn one (not for production): cf. GitSvnLoader + - git-svn one (not for production): cf. GitSvnSvnLoader - SWH one: cf. SWHSvnLoader + The main entry point of this is (no need to override it) + - def load(self): + + Inherit this class and then override the following functions: + - def build_swh_revision(self, rev, commit, dir_id, parents) + This is in charge of converting an svn revision to a compliant + swh revision + + - def process_repository(self) + This is in charge of processing the actual svn repository and + store the result to swh storage. + """ CONFIG_BASE_FILENAME = 'loader/svn.ini' def __init__(self, svn_url, destination_path, origin, with_svn_update=True): super().__init__(origin['id'], logging_class='swh.loader.svn.SvnLoader') self.with_svn_update = with_svn_update # noqa self.origin = origin def build_swh_revision(self, rev, commit, dir_id, parents): """Convert an svn revision to an swh one according to the loader's policy (git-svn or swh). Args: rev: the svn revision number commit: dictionary with keys 'author_name', 'author_date', 'rev', 'message' dir_id: the hash tree computation parents: the revision's parents Returns: The swh revision """ raise NotImplementedError('This should be overriden by subclass') - def check_history_not_altered(self, svnrepo, revision_start, swh_rev): - """Given a svn repository, check if the history was not tampered with. + def process_repository(self): + """The main idea of this function is to: + - iterate over the svn commit logs + - extract the svn commit log metadata + - compute the hashes from the current directory down to the + file + - compute the equivalent swh revision + - send all those objects for storage + - create an swh occurrence pointing to the last swh revision + seen + - send that occurrence for storage in swh-storage. """ - revision_id = swh_rev['id'] - parents = swh_rev['parents'] - hash_data_per_revs = svnrepo.swh_hash_data_at_revision(revision_start) - - rev = revision_start - rev, _, commit, objects_per_path = list(hash_data_per_revs)[0] - - dir_id = objects_per_path[b'']['checksums']['sha1_git'] - swh_revision = self.build_swh_revision(rev, - commit, - dir_id, - parents) - swh_revision_id = git.compute_revision_sha1_git(swh_revision) - - return swh_revision_id == revision_id + raise NotImplementedError('This should be implemented in subclass.') def process_svn_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process revisions from revision_start to revision_end and send to swh for storage. At each svn revision, checkout the repository, compute the tree hash and blobs and send for swh storage to store. Then computes and yields the swh revision. Yields: swh revision """ gen_revs = svnrepo.swh_hash_data_per_revision( revision_start, revision_end) for rev, nextrev, commit, objects_per_path in gen_revs: # compute the fs tree's checksums dir_id = objects_per_path[b'']['checksums']['sha1_git'] swh_revision = self.build_swh_revision( rev, commit, dir_id, revision_parents[rev]) swh_revision['id'] = git.compute_revision_sha1_git(swh_revision) self.log.debug('rev: %s, swhrev: %s, dir: %s' % ( rev, hashutil.hash_to_hex(swh_revision['id']), hashutil.hash_to_hex(dir_id))) if nextrev: revision_parents[nextrev] = [swh_revision['id']] self.maybe_load_contents( git.objects_per_type(GitType.BLOB, objects_per_path)) self.maybe_load_directories( git.objects_per_type(GitType.TREE, objects_per_path)) yield swh_revision def process_swh_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process and store revision to swh (sent by by blocks of 'revision_packet_size') Returns: The latest revision stored. """ for revisions in utils.grouper( self.process_svn_revisions(svnrepo, revision_start, revision_end, revision_parents), self.config['revision_packet_size']): revs = list(revisions) self.log.info('Processed %s revisions: [%s, ...]' % ( len(revs), hashutil.hash_to_hex(revs[0]['id']))) self.maybe_load_revisions(revs) return revs[-1] def process_swh_occurrence(self, revision, origin): """Process and load the occurrence pointing to the latest revision. """ occ = converters.build_swh_occurrence(revision['id'], origin['id'], datetime.datetime.utcnow()) self.log.debug('occ: %s' % occ) self.maybe_load_occurrences([occ]) - def process(self): + def load(self): """Load a svn repository in swh. Checkout the svn repository locally in destination_path. Args: - svn_url: svn repository url to import - origin: Dictionary origin - id: origin's id - url: url origin we fetched - type: type of the origin Returns: Dictionary with the following keys: - status: mandatory, the status result as a boolean - stderr: optional when status is True, mandatory otherwise """ - svnrepo = self.svnrepo - origin = self.origin try: - # default configuration - revision_start = 1 - revision_parents = { - revision_start: [] - } - - if self.with_svn_update: # Do we want to deal with update? - swh_rev = svnrepo.swh_previous_revision() - - if swh_rev: # Yes, we do. Try and update it. - extra_headers = dict(swh_rev['metadata']['extra_headers']) - revision_start = int(extra_headers['svn_revision']) - revision_parents = { - revision_start: swh_rev['parents'] - } - - self.log.debug('svn co %s@%s' % (svnrepo.remote_url, - revision_start)) - - if swh_rev and not self.check_history_not_altered( - svnrepo, - revision_start, - swh_rev): - msg = 'History of svn %s@%s history modified. Skipping...' % ( # noqa - svnrepo.remote_url, revision_start) - self.log.warn(msg) - return {'status': False, 'stderr': msg} - - revision_end = svnrepo.head_revision() - - self.log.info('[revision_start-revision_end]: [%s-%s]' % ( - revision_start, revision_end)) - - if revision_start == revision_end and revision_start is not 1: - self.log.info('%s@%s already injected.' % ( - svnrepo.remote_url, revision_end)) - return {'status': True} - - self.log.info('Processing %s.' % svnrepo) - - # process and store revision to swh (sent by by blocks of - # 'revision_packet_size') - latest_rev = self.process_swh_revisions(svnrepo, - revision_start, - revision_end, - revision_parents) - self.process_swh_occurrence(latest_rev, origin) - + self.process_repository() finally: # flush eventual remaining data self.flush() - svnrepo.clean_fs() + self.svnrepo.clean_fs() return {'status': True} class GitSvnSvnLoader(BaseSvnLoader): """Git-svn like loader (compute hashes a-la git-svn) Notes: This implementation is: - NOT for production - NOT able to deal with update. Default policy: Its default policy is to enrich (or even alter) information at each svn revision. It will: - truncate the timestamp of the svn commit date - alter the user to be an email using the repository's uuid as mailserver (user -> user@) - fills in the gap for empty author with '(no author)' name - remove empty folder (thus not counting them during hash computation) The equivalent git command is: `git svn clone -q --no-metadata` """ def __init__(self, svn_url, destination_path, origin): super().__init__(svn_url, destination_path, origin, with_svn_update=False) self.svnrepo = svn.GitSvnSvnRepo( svn_url, origin['id'], self.storage, destination_path=destination_path) def build_swh_revision(self, rev, commit, dir_id, parents): """Build the swh revision a-la git-svn. Args: rev: the svn revision commit: the commit metadata dir_id: the upper tree's hash identifier parents: the parents' identifiers Returns: The swh revision corresponding to the svn revision without any extra headers. """ return converters.build_gitsvn_swh_revision(rev, commit, dir_id, parents) + def process_repository(self): + """Load the repository's commits and send them for storage to swh. + + This does not deal with update. + + """ + origin = self.origin + svnrepo = self.svnrepo + # default configuration + revision_start = 1 + revision_parents = { + revision_start: [] + } + + revision_end = svnrepo.head_revision() + + self.log.info('[revision_start-revision_end]: [%s-%s]' % ( + revision_start, revision_end)) + + if revision_start == revision_end and revision_start is not 1: + self.log.info('%s@%s already injected.' % ( + svnrepo.remote_url, revision_end)) + return {'status': True} + + self.log.info('Processing %s.' % svnrepo) + + # process and store revision to swh (sent by by blocks of + # 'revision_packet_size') + latest_rev = self.process_swh_revisions(svnrepo, + revision_start, + revision_end, + revision_parents) + self.process_swh_occurrence(latest_rev, origin) + class SWHSvnLoader(BaseSvnLoader): """Swh svn loader is the main implementation destined for production. This implementation is able to deal with update on known svn repository. Default policy: It's to not add any information and be as close as possible from the svn data the server sent its way. The only thing that are added are the swh's revision 'extra_header' to be able to deal with update. """ def __init__(self, svn_url, destination_path, origin): super().__init__(svn_url, destination_path, origin) self.svnrepo = svn.SWHSvnRepo( svn_url, origin['id'], self.storage, destination_path=destination_path) + def check_history_not_altered(self, svnrepo, revision_start, swh_rev): + """Given a svn repository, check if the history was not tampered with. + + """ + revision_id = swh_rev['id'] + parents = swh_rev['parents'] + hash_data_per_revs = svnrepo.swh_hash_data_at_revision(revision_start) + + rev = revision_start + rev, _, commit, objects_per_path = list(hash_data_per_revs)[0] + + dir_id = objects_per_path[b'']['checksums']['sha1_git'] + swh_revision = self.build_swh_revision(rev, + commit, + dir_id, + parents) + swh_revision_id = git.compute_revision_sha1_git(swh_revision) + + return swh_revision_id == revision_id + def build_swh_revision(self, rev, commit, dir_id, parents): """Build the swh revision dictionary. This adds: - the 'synthetic' flag to true - the 'extra_headers' containing the repository's uuid and the svn revision number. Args: rev: the svn revision commit: the commit metadata dir_id: the upper tree's hash identifier parents: the parents' identifiers Returns: The swh revision corresponding to the svn revision. """ return converters.build_swh_revision(rev, commit, self.svnrepo.uuid, dir_id, parents) + + def process_repository(self): + svnrepo = self.svnrepo + origin = self.origin + + # default configuration + revision_start = 1 + revision_parents = { + revision_start: [] + } + + # Deal with update + swh_rev = svnrepo.swh_previous_revision() + + if swh_rev: # Yes, we do. Try and update it. + extra_headers = dict(swh_rev['metadata']['extra_headers']) + revision_start = int(extra_headers['svn_revision']) + revision_parents = { + revision_start: swh_rev['parents'] + } + + self.log.debug('svn co %s@%s' % (svnrepo.remote_url, + revision_start)) + + if swh_rev and not self.check_history_not_altered( + svnrepo, + revision_start, + swh_rev): + msg = 'History of svn %s@%s history modified. Skipping...' % ( # noqa + svnrepo.remote_url, revision_start) + self.log.warn(msg) + return {'status': False, 'stderr': msg} + + revision_end = svnrepo.head_revision() + + self.log.info('[revision_start-revision_end]: [%s-%s]' % ( + revision_start, revision_end)) + + if revision_start == revision_end and revision_start is not 1: + self.log.info('%s@%s already injected.' % ( + svnrepo.remote_url, revision_end)) + return {'status': True} + + self.log.info('Processing %s.' % svnrepo) + + # process and store revision to swh (sent by by blocks of + # 'revision_packet_size') + latest_rev = self.process_swh_revisions(svnrepo, + revision_start, + revision_end, + revision_parents) + self.process_swh_occurrence(latest_rev, origin) diff --git a/swh/loader/svn/svn.py b/swh/loader/svn/svn.py index 4a28fb0..dcbc2bb 100644 --- a/swh/loader/svn/svn.py +++ b/swh/loader/svn/svn.py @@ -1,397 +1,395 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """SVN client in charge of iterating over svn logs and yield commit representations including the hash tree/content computations per svn commit. """ import os import tempfile import shutil from subvertpy.ra import RemoteAccess, Auth, get_username_provider from subvertpy import client, properties from swh.model import git from . import ra, utils, converters # When log message contains empty data DEFAULT_AUTHOR_MESSAGE = '' class SvnRepoException(ValueError): def __init__(self, svnrepo, e): super().__init__(e) self.svnrepo = svnrepo class BaseSvnRepo(): """Base svn repository representation for swh. To override some of the behavior regarding the message log properties, you can instantiate a subclass of this class and override: - def convert_commit_author(self, author) - def convert_commit_message(self, msg) - def convert_commit_date(self, date) cf. GitSvnSvnRepo, SwhSvnRepo for instanciation example. """ def __init__(self, remote_url, origin_id, storage, destination_path=None): self.remote_url = remote_url.rstrip('/') self.storage = storage self.origin_id = origin_id if destination_path: os.makedirs(destination_path, exist_ok=True) root_dir = destination_path else: root_dir = '/tmp' self.local_dirname = tempfile.mkdtemp(suffix='.swh.loader', prefix='tmp.', dir=root_dir) local_name = os.path.basename(self.remote_url) auth = Auth([get_username_provider()]) # one connection for log iteration self.conn_log = RemoteAccess(self.remote_url, auth=auth) # another for replay self.conn = RemoteAccess(self.remote_url, auth=auth) # one client for update operation self.client = client.Client(auth=auth) self.local_url = os.path.join(self.local_dirname, local_name).encode( 'utf-8') self.uuid = self.conn.get_uuid().encode('utf-8') def __str__(self): return str({'remote_url': self.remote_url, 'local_url': self.local_url, 'uuid': self.uuid, 'swh-origin': self.origin_id}) def head_revision(self): """Retrieve current revision of the repository's working copy. """ return self.conn.get_latest_revnum() def initial_revision(self): """Retrieve the initial revision from which the remote url appeared. Note: This should always be 1 since we won't be dealing with in-depth url. """ return 1 def convert_commit_message(self, msg): """Do something with message (e.g add extra line, etc...) cf. SvnRepo for a simple implementation. Args: msg (str): the commit message to convert. Returns: The transformed message as bytes. """ raise NotImplementedError('Should be overridden by subclass.') def convert_commit_date(self, date): """Convert the message date (e.g, convert into timestamp or whatever makes sense to you.). Args: date (str): the commit date to convert. Returns: The transformed date. """ raise NotImplementedError('Should be overridden by subclass.') def convert_commit_author(self, author): """Convert the commit author (e.g, convert into dict or whatever makes sense to you.). Args: author (str): the commit author to convert. Returns: The transformed author as dict. """ raise NotImplementedError('Should be overridden by subclass.') def __to_entry(self, log_entry): changed_paths, rev, revprops, has_children = log_entry author_date = self.convert_commit_date( revprops.get(properties.PROP_REVISION_DATE)) author = self.convert_commit_author( revprops.get(properties.PROP_REVISION_AUTHOR)) message = self.convert_commit_message( revprops.get(properties.PROP_REVISION_LOG, DEFAULT_AUTHOR_MESSAGE)) return { 'rev': rev, 'author_date': author_date, 'author_name': author, 'message': message, } def logs(self, revision_start, revision_end): """Stream svn logs between revision_start and revision_end by chunks of block_size logs. Yields revision and associated revision information between the revision start and revision_end. Args: revision_start: the svn revision starting bound revision_end: the svn revision ending bound Yields: tuple of revisions and logs. revisions: list of revisions in order logs: Dictionary with key revision number and value the log entry. The log entry is a dictionary with the following keys: - author_date: date of the commit - author_name: name of the author - message: commit message """ for log_entry in self.conn_log.iter_log(paths=None, start=revision_start, end=revision_end, discover_changed_paths=False): yield self.__to_entry(log_entry) def export(self, revision): """Export the repository to a given version. """ self.client.export(self.remote_url, to=self.local_url.decode('utf-8'), rev=revision) def swh_previous_revision(self): """Look for possible existing revision in swh. Returns: The previous swh revision if found, None otherwise. """ storage = self.storage occ = storage.occurrence_get(self.origin_id) if occ: revision_id = occ[0]['target'] revisions = storage.revision_get([revision_id]) if revisions: return revisions[0] def swh_hash_data_per_revision(self, start_revision, end_revision): """Compute swh hash data per each revision between start_revision and end_revision. Args: start_revision: starting revision end_revision: ending revision Yields: tuple (rev, nextrev, commit, objects_per_path) - rev: current revision - nextrev: next revision - commit: commit data (author, date, message) for such revision - objects_per_path: dictionary of path, swh hash data with type """ hashes = {} for commit in self.logs(start_revision, end_revision): rev = commit['rev'] hashes = self.swhreplay.compute_hashes(rev) if rev == end_revision: nextrev = None else: nextrev = rev + 1 yield rev, nextrev, commit, hashes def swh_hash_data_at_revision(self, revision): """Compute the hash data at revision. Expected to be used for update only. """ # Update the disk at revision self.export(revision) # Compute the current hashes on disk - hashes = git.compute_hashes_from_directory( - self.local_url, - remove_empty_folder=not self.with_empty_folder) + hashes = git.compute_hashes_from_directory(self.local_url) hashes = utils.convert_hashes_with_relative_path( hashes, rootpath=self.local_url) # Update the replay collaborator with the right state self.swhreplay = ra.SWHReplay( conn=self.conn, rootpath=self.local_url, state=hashes) # Retrieve the commit information for revision commit = list(self.logs(revision, revision))[0] yield revision, revision + 1, commit, hashes def clean_fs(self): """Clean up the local working copy. """ shutil.rmtree(self.local_dirname) class GitSvnSvnRepo(BaseSvnRepo): """Svn repository mapping a-la git-svn. This class does exactly as BaseSvnRepo except for: - the commit message which is extended with a new line and then encoded - the commit author is converted using the repository's uuid for his/her email - the commit date is transformed into timestamp and truncated - Extra commit line in commit message. - user@ in raw commit message - truncated timestamp in raw commit message """ def __init__(self, remote_url, origin_id, storage, destination_path=None): super().__init__(remote_url, origin_id, storage, destination_path) self.swhreplay = ra.SWHReplayNoEmptyFolder( conn=self.conn, rootpath=self.local_url) self.with_empty_folder = False def convert_commit_message(self, msg): """Add an extra line to the commit message and encode it in utf-8. Args: msg (str): the commit message to convert. Returns: The transformed message as bytes. """ return ('%s\n' % msg).encode('utf-8') def convert_commit_date(self, date): """Convert the commit message date into truncated timestamp in swh format. Args: date (str): the commit date to convert. Returns: The transformed date. """ return converters.svn_date_to_gitsvn_date(date) def convert_commit_author(self, author): """Convert the commit author into an swh person. The user becomes a dictionary of the form: { name: author, email: author@repo-uuid, fullname: author } If the user is already some kind of fullname, this is what is used as fullname. Args: author (str): the commit author to convert. Returns: The transformed author as dict. """ return converters.svn_author_to_gitsvn_person(author, self.uuid) class SWHSvnRepo(BaseSvnRepo): """This class does exactly as BaseSvnRepo except for: - the commit message which is simply encoded - the commit author is left as is. - the commit timestamp is left as is. """ def __init__(self, remote_url, origin_id, storage, destination_path=None): super().__init__(remote_url, origin_id, storage, destination_path) self.swhreplay = ra.SWHReplay( conn=self.conn, rootpath=self.local_url) def convert_commit_message(self, msg): """Simply encode the commit message. Args: msg (str): the commit message to convert. Returns: The transformed message as bytes. """ return msg.encode('utf-8') def convert_commit_date(self, date): """Convert the message commit date into a timestamp in swh format. The precision is kept. Args: date (str): the commit date to convert. Returns: The transformed date. """ return converters.svn_date_to_swh_date(date) def convert_commit_author(self, author): """Convert the commit author into an swh person. The user becomes a dictionary of the form: { name: author, email: '', fullname: author } Args: author (str): the commit author to convert. Returns: The transformed author as dict. """ return converters.svn_author_to_swh_person(author) diff --git a/swh/loader/svn/tasks.py b/swh/loader/svn/tasks.py index 532f06a..cee5766 100644 --- a/swh/loader/svn/tasks.py +++ b/swh/loader/svn/tasks.py @@ -1,51 +1,49 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.loader.core import tasks from swh.loader.svn.loader import GitSvnSvnLoader, SWHSvnLoader class LoadSvnRepositoryTsk(tasks.LoaderCoreTask): """Import one svn repository to Software Heritage. """ CONFIG_BASE_FILENAME = 'loader/svn.ini' ADDITIONAL_CONFIG = { 'storage_class': ('str', 'remote_storage'), 'storage_args': ('list[str]', ['http://localhost:5000/']), 'with_policy': ('string', 'swh'), # Default, other possible # value is 'gitsvn' - 'with_svn_update': ('bool', True), } task_queue = 'swh_loader_svn' def run(self, svn_url, local_path): """Import a svn repository. Args: cf. swh.loader.svn.SvnLoader.process docstring """ - print(self.config) origin = {'type': 'svn', 'url': svn_url} origin['id'] = self.storage.origin_add_one(origin) fetch_history_id = self.open_fetch_history(origin['id']) # Determine which loader to trigger if self.config['with_policy'] == 'gitsvn': loader = GitSvnSvnLoader(svn_url, local_path, origin) elif self.config['with_policy'] == 'swh': loader = SWHSvnLoader(svn_url, local_path, origin) else: raise ValueError('Only gitsvn or swh policies are supported in' '\'with_policy\' entry. ' 'Please adapt your svn.ini file accordingly') - result = loader.process() + result = loader.load() self.close_fetch_history(fetch_history_id, result)