diff --git a/PKG-INFO b/PKG-INFO index 0034acc..754511a 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.loader.svn -Version: 0.0.23 +Version: 0.0.24 Summary: Software Heritage Loader SVN Home-page: https://forge.softwareheritage.org/diffusion/DLDSVN Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/README b/README index 1720f36..9b1995e 100644 --- a/README +++ b/README @@ -1,131 +1,107 @@ swh-loader-svn ============== Documents are in the ./docs folder: - Specification: ./docs/swh-loader-svn.txt - Comparison performance with git-svn: ./docs/comparison-git-svn-swh-svn.org # Configuration file ## Location Either: - /etc/softwareheritage/loader/svn.ini - ~/.config/swh/loader/svn.ini - ~/.swh/loader/svn.ini -## Default options +## Configuration sample ``` -[main] -storage_class = remote_storage -storage_args = http://localhost:5000/ - -send_contents = True -send_directories = True -send_revisions = True -send_releases = True -send_occurrences = True +storage: + cls: remote + args: + url: http://localhost:5002/ + +send_contents: true +send_directories: true +send_revisions: true +send_releases: true +send_occurrences: true # nb of max contents to send for storage -content_packet_size = 100 +content_packet_size: 10000 # 100 Mib of content data -content_packet_block_size_bytes = 104857600 +content_packet_block_size_bytes: 104857600 # limit for swh content storage for one blob (beyond that limit, the # content's data is not sent for storage) -content_packet_size_bytes = 1073741824 -directory_packet_size = 2500 -revision_packet_size = 1000 -release_packet_size = 1 -occurrence_packet_size = 1 - -# Flags -with_policy = swh -``` - -This policy does not: -- alter anything in regards of svn data -- adds an extra-headers with repo uuid and svn revision. This permits to deal with update. +content_packet_size_bytes: 1073741824 +directory_packet_size: 2500 +revision_packet_size: 10 +release_packet_size: 1000 +occurrence_packet_size: 1000 -## git-svn like - -For a git-svn like (ignore empty folder, no extra metadata, adapt -user's name with repository's uuid, add extra commit line, no update -policy), adapt to the previous options with the following: - -```txt -with_policy = gitsvn +check_revision: 10 ``` -Notes: - -This policy does alter things: -- it adds the repo-uuid on author's name (author@) -- truncates date time commits so some precision is lost on date -- removes empty folder if any are checkouted on disk - -Those modifications, in effects, alter the git hash computation of -revisions. - -This policy is destined for test. Thus: -- no persistence takes place (even if send_* options are True) -- no update is possible - - -# Starting one instance - -## worker's configuration file - -The file is either at: -- /etc/softwareheritage/worker.ini -- ~/.config/swh/worker.ini -- ~/.swh/worker.ini - ## configuration content With at least the following module (swh.loader.svn.tasks) and queue (swh_loader_svn): ``` [main] task_broker = amqp://guest@localhost// -task_modules = swh.loader.svn.tasks,swh.loader.tar.tasks -task_queues = swh_loader_svn, swh_loader_tar +task_modules = swh.loader.svn.tasks +task_queues = swh_loader_svn task_soft_time_limit = 0 ``` swh.loader.svn.tasks and swh_loader_svn are the important entries here. ## start worker instance To start a current worker instance: ```sh python3 -m celery worker --app=swh.scheduler.celery_backend.config.app \ --pool=prefork \ --concurrency=10 \ -Ofair \ --loglevel=debug 2>&1 ``` ## Produce a repository to load -Either one repository: +You can see: + +`python3 -m swh.loader.svn.producer svn --help` + +### one repository ```sh -python3 -u -m swh.loader.svn.producer --svn-url file:///home/storage/svn/repos/pkg-fox +python3 -u -m swh.loader.svn.producer svn --svn-url file:///home/storage/svn/repos/pkg-fox --visit-date 'Tue, 3 May 2017 17:16:32 +0200' ``` -or a bunch: +Note: +- `--visit-date` to override the default visit-date to now. + +### multiple repositories + ```sh -cat ~/svn-repository-list | python3 -m swh.loader.svn.producer +cat ~/svn-repository-list | python3 -m swh.loader.svn.producer svn ``` -svn-repository is a list of svn repository urls (one per line). -Something like: +The file svn-repository-list contains a list of svn repository urls +(one per line), something like: + ```txt -svn://svn.debian.org/svn/pkg-fox/ -svn://svn.debian.org/svn/glibc-bsd/ -svn://svn.debian.org/svn/pkg-voip/ -svn://svn.debian.org/svn/python-modules/ -svn://svn.debian.org/svn/pkg-gnome/ +svn://svn.debian.org/svn/pkg-fox/ optional-url +svn://svn.debian.org/svn/glibc-bsd/ optional-url +svn://svn.debian.org/svn/pkg-voip/ optional-url +svn://svn.debian.org/svn/python-modules/ optional-url +svn://svn.debian.org/svn/pkg-gnome/ optional-url ``` + +## Produce archive of svndumps list to load + +see. `python3 -m swh.loader.svn.producer svn-archive --help` + diff --git a/debian/control b/debian/control index 9519f40..2233960 100644 --- a/debian/control +++ b/debian/control @@ -1,35 +1,35 @@ Source: swh-loader-svn Maintainer: Software Heritage developers Section: python Priority: optional Build-Depends: debhelper (>= 9), dh-python, python3-all, python3-click, python3-dateutil, python3-nose, python3-retrying, python3-setuptools, python3-subvertpy (>= 0.9.4~), python3-swh.core (>= 0.0.19~), - python3-swh.loader.core (>= 0.0.12~), + python3-swh.loader.core (>= 0.0.13~), python3-swh.model (>= 0.0.11~), python3-swh.scheduler (>= 0.0.11~), python3-swh.storage (>= 0.0.79~), python3-vcversioner Standards-Version: 3.9.6 Homepage: https://forge.softwareheritage.org/diffusion/DLDSVN/ Package: python3-swh.loader.svn Architecture: all Depends: pigz, python3-swh.core (>= 0.0.19~), - python3-swh.loader.core (>= 0.0.12~), + python3-swh.loader.core (>= 0.0.13~), python3-swh.model (>= 0.0.11~), python3-swh.scheduler (>= 0.0.11~), python3-swh.storage (>= 0.0.79~), subversion, ${misc:Depends}, ${python3:Depends} Description: Software Heritage Loader Svn Module in charge of loading svn repositories into swh storage. diff --git a/requirements-swh.txt b/requirements-swh.txt index fd95219..e7b3fc5 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,5 +1,5 @@ swh.core >= 0.0.19 swh.storage >= 0.0.79 swh.model >= 0.0.11 swh.scheduler >= 0.0.11 -swh.loader.core >= 0.0.12 +swh.loader.core >= 0.0.13 diff --git a/resources/svn.ini b/resources/svn.ini index f834380..8da9f00 100644 --- a/resources/svn.ini +++ b/resources/svn.ini @@ -1,24 +1,24 @@ [main] storage_class = remote_storage -storage_args = http://localhost:5000/ +storage_args = http://localhost:5002/ send_contents = True send_directories = True send_revisions = True send_releases = True send_occurrences = True # nb of max contents to send for storage (if size threshold not reached before) content_packet_size = 10000 # 100 Mib of content data (size threshold of data before sending for storage) content_packet_block_size_bytes = 104857600 # limit for swh content storage for one blob (beyond that limit, the # content's data is not sent for storage) content_packet_size_bytes = 1073741824 # packet of directories to send for storage directory_packet_size = 25000 # packet of revisions to send for storage revision_packet_size = 10000 # packet of releases to send for storage release_packet_size = 100000 # packet of occurrences to send for storage occurrence_packet_size = 100000 diff --git a/swh.loader.svn.egg-info/PKG-INFO b/swh.loader.svn.egg-info/PKG-INFO index 0034acc..754511a 100644 --- a/swh.loader.svn.egg-info/PKG-INFO +++ b/swh.loader.svn.egg-info/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.loader.svn -Version: 0.0.23 +Version: 0.0.24 Summary: Software Heritage Loader SVN Home-page: https://forge.softwareheritage.org/diffusion/DLDSVN Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/swh.loader.svn.egg-info/requires.txt b/swh.loader.svn.egg-info/requires.txt index 83fe043..457a2a3 100644 --- a/swh.loader.svn.egg-info/requires.txt +++ b/swh.loader.svn.egg-info/requires.txt @@ -1,10 +1,10 @@ click python-dateutil retrying subvertpy>=0.9.4 swh.core>=0.0.19 -swh.loader.core>=0.0.12 +swh.loader.core>=0.0.13 swh.model>=0.0.11 swh.scheduler>=0.0.11 swh.storage>=0.0.79 vcversioner diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py index c524eb7..5ed2444 100644 --- a/swh/loader/svn/loader.py +++ b/swh/loader/svn/loader.py @@ -1,539 +1,502 @@ -# Copyright (C) 2015-2016 The Software Heritage developers +# Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Loader in charge of injecting either new or existing svn mirrors to swh-storage. """ import abc -import datetime +import os +import shutil from swh.core import utils, hashutil from swh.model import git from swh.model.git import GitType from swh.loader.core.loader import SWHLoader from . import svn, converters -from .utils import hashtree +from .utils import hashtree, init_svn_repo_from_archive_dump class SvnLoaderEventful(ValueError): """A wrapper exception to transit the swh_revision onto which the loading failed. """ def __init__(self, e, swh_revision): super().__init__(e) self.swh_revision = swh_revision class SvnLoaderUneventful(ValueError): pass class SvnLoaderHistoryAltered(ValueError): pass class BaseSvnLoader(SWHLoader, metaclass=abc.ABCMeta): """Base Svn loader to load one svn repository according to specific policies (only swh one now). The main entry point of this is (no need to override it) - def load(self, origin_visit, last_known_swh_revision=None): pass Inherit this class and then override the following functions: - def build_swh_revision(self, rev, commit, dir_id, parents) This is in charge of converting an svn revision to a compliant swh revision - def process_repository(self) This is in charge of processing the actual svn repository and store the result to swh storage. """ CONFIG_BASE_FILENAME = 'loader/svn' ADDITIONAL_CONFIG = { 'check_revision': ('int', 1000), } def __init__(self): super().__init__(logging_class='swh.loader.svn.SvnLoader') self.check_revision = self.config['check_revision'] @abc.abstractmethod def swh_revision_hash_tree_at_svn_revision(self, revision): """Compute and return the hash tree at a given svn revision. Args: rev (int): the svn revision we want to check Returns: The hash tree directory as bytes. """ pass @abc.abstractmethod def get_svn_repo(self, svn_url, destination_path, origin): """Instantiates the needed svnrepo collaborator to permit reading svn repository. Args: svn_url: the svn repository url to read from destination_path: the local path on disk to compute data origin: the corresponding origin Returns: Instance of swh.loader.svn.svn clients """ raise NotImplementedError @abc.abstractmethod def build_swh_revision(self, rev, commit, dir_id, parents): """Convert an svn revision to an swh one according to the loader's policy (git-svn or swh). Args: rev: the svn revision number commit: dictionary with keys 'author_name', 'author_date', 'rev', 'message' dir_id: the hash tree computation parents: the revision's parents Returns: The swh revision """ raise NotImplementedError @abc.abstractmethod def process_repository(self, origin_visit, last_known_swh_revision=None): """The main idea of this function is to: - iterate over the svn commit logs - extract the svn commit log metadata - compute the hashes from the current directory down to the file - compute the equivalent swh revision - send all those objects for storage - create an swh occurrence pointing to the last swh revision seen - send that occurrence for storage in swh-storage. """ raise NotImplementedError def process_svn_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process revisions from revision_start to revision_end and send to swh for storage. At each svn revision, checkout the repository, compute the tree hash and blobs and send for swh storage to store. Then computes and yields the swh revision. Note that at every self.check_revision, an svn export is done and a hash tree is computed to check that no divergence occurred. Yields: swh revision as a dictionary with keys, sha1_git, sha1, etc... """ gen_revs = svnrepo.swh_hash_data_per_revision( revision_start, revision_end) swh_revision = None count = 0 for rev, nextrev, commit, objects_per_path in gen_revs: count += 1 # Send the associated contents/directories self.maybe_load_contents( git.objects_per_type(GitType.BLOB, objects_per_path)) self.maybe_load_directories( git.objects_per_type(GitType.TREE, objects_per_path)) # compute the fs tree's checksums dir_id = objects_per_path[b'']['checksums']['sha1_git'] swh_revision = self.build_swh_revision( rev, commit, dir_id, revision_parents[rev]) swh_revision['id'] = git.compute_revision_sha1_git( swh_revision) self.log.debug('rev: %s, swhrev: %s, dir: %s' % ( rev, hashutil.hash_to_hex(swh_revision['id']), hashutil.hash_to_hex(dir_id))) if (count % self.check_revision) == 0: # hash computation check self.log.info('Checking hash computations on revision %s...' % rev) checked_dir_id = self.swh_revision_hash_tree_at_svn_revision( rev) if checked_dir_id != dir_id: err = 'Hash tree computation divergence detected (%s != %s), stopping!' % ( # noqa hashutil.hash_to_hex(dir_id), hashutil.hash_to_hex(checked_dir_id) ) raise ValueError(err) if nextrev: revision_parents[nextrev] = [swh_revision['id']] yield swh_revision def process_swh_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process and store revision to swh (sent by blocks of 'revision_packet_size') Returns: The latest revision stored. """ try: swh_revision_gen = self.process_svn_revisions(svnrepo, revision_start, revision_end, revision_parents) revs = [] for revisions in utils.grouper( swh_revision_gen, self.config['revision_packet_size']): revs = list(revisions) self.log.info('Processed %s revisions: [%s, ...]' % ( len(revs), hashutil.hash_to_hex(revs[0]['id']))) self.maybe_load_revisions(revs) except Exception as e: if revs: # flush remaining revisions self.maybe_load_revisions(revs) # Take the last one as the last known revisions known_swh_rev = revs[-1] # Then notify something is wrong, and we stopped at that rev. raise SvnLoaderEventful(e, swh_revision={ 'id': known_swh_rev['id'], }) raise e return revs[-1] def process_swh_occurrence(self, revision, origin_visit): """Process and load the occurrence pointing to the latest revision. """ - occ = converters.build_swh_occurrence(revision['id'], - origin_visit['origin'], - origin_visit['visit']) + occ = converters.build_swh_occurrence( + revision['id'], origin_visit['origin'], origin_visit['visit']) self.log.debug('occ: %s' % occ) self.maybe_load_occurrences([occ]) - def close_success(self): - self.close_fetch_history_success(self.fetch_history_id) - self.storage.origin_visit_update(self.origin_visit['origin'], - self.origin_visit['visit'], - status='full') - - def close_failure(self): - self.close_fetch_history_failure(self.fetch_history_id) - self.storage.origin_visit_update(self.origin_visit['origin'], - self.origin_visit['visit'], - status='partial') - - def load(self, *args, **kwargs): - """Load a svn repository in swh. - - Checkout the svn repository locally in destination_path. - - Args: - - origin_visit: (mandatory) The current origin visit - - last_known_swh_revision: (Optional) Hash id of known swh revision - already visited in a previous visit - - Returns: - Dictionary with the following keys: - - eventful: (mandatory) is the loading being eventful or not - - completion: (mandatory) 'full' if complete, 'partial' otherwise - - state: (optional) if the completion was partial, this - gives the state to pass along for the next schedule - - """ - self.prepare(*args, **kwargs) - try: - latest_rev = self.process_repository( - self.origin_visit, self.last_known_swh_revision) - except SvnLoaderEventful as e: - self.log.error('Eventful partial visit. Detail: %s' % e) - latest_rev = e.swh_revision - self.process_swh_occurrence(latest_rev, self.origin_visit) - self.close_failure() - return { - 'eventful': True, - 'completion': 'partial', - 'state': { - 'origin': self.origin_visit['origin'], - 'revision': hashutil.hash_to_hex(latest_rev['id']) - } - } - except (SvnLoaderHistoryAltered, SvnLoaderUneventful) as e: - self.log.error('Uneventful visit. Detail: %s' % e) - # FIXME: This fails because latest_rev is not bound - # self.process_swh_occurrence(latest_rev, self.origin_visit) - self.close_failure() - return { - 'eventful': False, - } - except Exception as e: - self.close_failure() - raise e - else: - self.process_swh_occurrence(latest_rev, self.origin_visit) - self.close_success() - return { - 'eventful': True, - 'completion': 'full', - } - finally: - self.clean() - - @abc.abstractmethod - def clean(self): - """Clean up after working. - - """ - pass - def prepare(self, *args, **kwargs): - """ - Prepare origin, fetch_origin, origin_visit - Then load a svn repository. - - Then close origin_visit, fetch_history according to status success or - failure. - - First: - - creates an origin if it does not exist - - creates a fetch_history entry - - creates an origin_visit - - Then loads the svn repository - - """ destination_path = kwargs['destination_path'] # local svn url svn_url = kwargs['svn_url'] origin_url = kwargs.get('origin_url') - visit_date = kwargs.get('visit_date') + self.visit_date = kwargs.get('visit_date') origin = { 'url': origin_url if origin_url else svn_url, 'type': 'svn', } - if 'origin' not in kwargs: # first time, we'll create the origin - origin['id'] = self.storage.origin_add_one(origin) - else: - origin['id'] = kwargs['origin'] - - self.origin_id = origin['id'] + self.origin_id = self.send_origin(origin) + origin['id'] = self.origin_id + self.origin = origin if 'swh_revision' in kwargs: self.last_known_swh_revision = hashutil.hex_to_hash( kwargs['swh_revision']) else: self.last_known_swh_revision = None self.svnrepo = self.get_svn_repo(svn_url, destination_path, origin) self.fetch_history_id = self.open_fetch_history() - if not visit_date: - visit_date = datetime.datetime.now(tz=datetime.timezone.utc) + def get_origin(self): + """Retrieve the origin we are working with. - self.origin_visit = self.storage.origin_visit_add( - self.origin_id, visit_date) + """ + return self.origin # set in prepare method + + def fetch_data(self): + """We need to fetch and stream the data to store directly. So + fetch_data do actually nothing. The method `store_data` below + is in charge to do everything, fetch and store. + + """ + pass + + def store_data(self): + """We need to fetch and stream the data to store directly because + there is too much data and state changes. Everything is + intertwined together (We receive patch and apply on disk and + compute at the hashes at the same time) + + So every data to fetch and store is done here. + + Requisite: origin_visit and last_known_swh_revision must have + been set in the prepare method. + + """ + origin_visit = {'origin': self.origin_id, 'visit': self.visit} + try: + latest_rev = self.process_repository(origin_visit, + self.last_known_swh_revision) + except SvnLoaderEventful as e: + self.log.error('Eventful partial visit. Detail: %s' % e) + latest_rev = e.swh_revision + self.process_swh_occurrence(latest_rev, origin_visit) + raise + except (SvnLoaderHistoryAltered, SvnLoaderUneventful) as e: + self.log.error('Uneventful visit. Detail: %s' % e) + raise + except Exception as e: + raise + else: + self.process_swh_occurrence(latest_rev, origin_visit) class SWHSvnLoader(BaseSvnLoader): """Swh svn loader is the main implementation destined for production. This implementation is able to deal with update on known svn repository. Default policy: Keep data as close as possible from the original svn data. We only add information that are needed for update or continuing from last known revision (svn revision and svn repository's uuid). """ def __init__(self): super().__init__() - def clean(self): + def cleanup(self): """Clean after oneself. This is in charge to flush the remaining data to write in swh storage. And to clean up the svn repository's working representation on disk. """ - self.flush() self.svnrepo.clean_fs() def swh_revision_hash_tree_at_svn_revision(self, revision): """Compute a given hash tree at specific revision. """ local_dirname, local_url = self.svnrepo.export_temporary(revision) h = hashtree(local_url)['sha1_git'] self.svnrepo.clean_fs(local_dirname) return h def get_svn_repo(self, svn_url, destination_path, origin): return svn.SWHSvnRepo( svn_url, origin['id'], self.storage, destination_path=destination_path) def swh_previous_revision(self, prev_swh_revision=None): """Retrieve swh's previous revision if any. """ return self.svnrepo.swh_previous_revision(prev_swh_revision) def check_history_not_altered(self, svnrepo, revision_start, swh_rev): """Given a svn repository, check if the history was not tampered with. """ revision_id = swh_rev['id'] parents = swh_rev['parents'] hash_data_per_revs = svnrepo.swh_hash_data_at_revision(revision_start) rev = revision_start rev, _, commit, objects_per_path = list(hash_data_per_revs)[0] dir_id = objects_per_path[b'']['checksums']['sha1_git'] swh_revision = self.build_swh_revision(rev, commit, dir_id, parents) swh_revision_id = git.compute_revision_sha1_git(swh_revision) return swh_revision_id == revision_id def build_swh_revision(self, rev, commit, dir_id, parents): """Build the swh revision dictionary. This adds: - the 'synthetic' flag to true - the 'extra_headers' containing the repository's uuid and the svn revision number. Args: rev: the svn revision commit: the commit metadata dir_id: the upper tree's hash identifier parents: the parents' identifiers Returns: The swh revision corresponding to the svn revision. """ return converters.build_swh_revision(rev, commit, self.svnrepo.uuid, dir_id, parents) def init_from(self, partial_swh_revision, previous_swh_revision): """Function to determine from where to start from. Args: - partial_swh_revision: A known revision from which the previous loading did not finish. - known_previous_revision: A known revision from which the previous loading did finish. Returns: The revision from which to start or None if nothing (fresh start). """ if partial_swh_revision and not previous_swh_revision: return partial_swh_revision if not partial_swh_revision and previous_swh_revision: return previous_swh_revision if partial_swh_revision and previous_swh_revision: # will determine from which to start from extra_headers1 = dict( partial_swh_revision['metadata']['extra_headers']) extra_headers2 = dict( previous_swh_revision['metadata']['extra_headers']) rev_start1 = int(extra_headers1['svn_revision']) rev_start2 = int(extra_headers2['svn_revision']) if rev_start1 <= rev_start2: return previous_swh_revision return partial_swh_revision return None def process_repository(self, origin_visit, last_known_swh_revision=None): svnrepo = self.svnrepo # default configuration revision_start = 1 revision_parents = { revision_start: [] } # Check if we already know a previous revision for that origin swh_rev = self.swh_previous_revision() # Determine from which known revision to start swh_rev = self.init_from(last_known_swh_revision, previous_swh_revision=swh_rev) if swh_rev: # Yes, we do. Try and update it. extra_headers = dict(swh_rev['metadata']['extra_headers']) revision_start = int(extra_headers['svn_revision']) revision_parents = { revision_start: swh_rev['parents'], } self.log.debug('svn export --ignore-keywords %s@%s' % ( svnrepo.remote_url, revision_start)) if swh_rev and not self.check_history_not_altered( svnrepo, revision_start, swh_rev): msg = 'History of svn %s@%s history modified. Skipping...' % ( svnrepo.remote_url, revision_start) self.log.warn(msg) raise SvnLoaderHistoryAltered else: # now we know history is ok, we start at next revision revision_start = revision_start + 1 # and the parent become the latest know revision for # that repository revision_parents[revision_start] = [swh_rev['id']] revision_end = svnrepo.head_revision() self.log.info('[revision_start-revision_end]: [%s-%s]' % ( revision_start, revision_end)) if revision_start > revision_end and revision_start is not 1: self.log.info('%s@%s already injected.' % ( svnrepo.remote_url, revision_end)) raise SvnLoaderUneventful self.log.info('Processing %s.' % svnrepo) # process and store revision to swh (sent by by blocks of # 'revision_packet_size') return self.process_swh_revisions( svnrepo, revision_start, revision_end, revision_parents) + + +class SWHSvnLoaderFromDumpArchive(SWHSvnLoader): + """Load a svn repository from an archive (containing a dump). + + """ + def __init__(self, archive_path): + super().__init__() + self.log.info('Archive to mount and load %s' % archive_path) + self.temp_dir, self.repo_path = init_svn_repo_from_archive_dump( + archive_path) + + def cleanup(self): + super().cleanup() + + if self.temp_dir and os.path.exists(self.temp_dir): + self.log.debug('Clean up temp directory %s for project %s' % ( + self.temp_dir, os.path.basename(self.repo_path))) + shutil.rmtree(self.temp_dir) diff --git a/swh/loader/svn/producer.py b/swh/loader/svn/producer.py index aa15a8c..7c0c16b 100644 --- a/swh/loader/svn/producer.py +++ b/swh/loader/svn/producer.py @@ -1,106 +1,108 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import click import sys from swh.scheduler.utils import get_task def _produce_svn_to_load( svn_url, origin_url, destination_path=None, visit_date=None, synchroneous=False, task_name='swh.loader.svn.tasks.LoadSWHSvnRepositoryTsk'): """Produce svn urls on the message queue. Those urls can either be read from stdin or directly passed as argument. """ task = get_task(task_name) if not synchroneous and svn_url: task.delay(svn_url=svn_url, origin_url=origin_url, visit_date=visit_date, destination_path=destination_path) elif synchroneous and svn_url: # for debug purpose task(svn_url=svn_url, origin_url=origin_url, visit_date=visit_date, destination_path=destination_path) else: # input from stdin, so we ignore most of the function's input for line in sys.stdin: line = line.rstrip() data = line.split(' ') svn_url = data[0] if len(data) > 1: origin_url = data[1] else: origin_url = None if svn_url: print(svn_url, origin_url) task.delay(svn_url=svn_url, origin_url=origin_url, + visit_date=visit_date, destination_path=destination_path) def _produce_archive_to_mount_and_load( archive_path, visit_date, task_name='swh.loader.svn.tasks.MountAndLoadSvnRepositoryTsk'): task = get_task(task_name) if archive_path: task.delay(archive_path) else: for line in sys.stdin: line = line.rstrip() data = line.split(' ') archive_path = data[0] if len(data) > 1: origin_url = data[1] else: origin_url = None if archive_path: print(archive_path, origin_url) task.delay(archive_path, origin_url, visit_date=visit_date) @click.group() def cli(): pass @cli.command('svn', help='Default svn urls producer') @click.option('--url', help="svn repository's mirror url.") @click.option('--origin-url', default=None, help='svn repository\'s original remote url ' '(if different than --svn-url).') @click.option('--destination-path', help="(optional) svn checkout destination.") @click.option('--visit-date', help="(optional) visit date to override") @click.option('--synchroneous', is_flag=True, help="To execute directly the svn loading.") def produce_svn_to_load(url, origin_url, destination_path, visit_date, synchroneous): _produce_svn_to_load(svn_url=url, origin_url=origin_url, + visit_date=visit_date, destination_path=destination_path, synchroneous=synchroneous) @cli.command('svn-archive', help='Default svndump archive producer') @click.option('--visit-date', help="(optional) visit date to override") @click.option('--path', help="Archive's Path to load and mount") def produce_archive_to_mount_and_load(path, visit_date): _produce_archive_to_mount_and_load(path, visit_date) if __name__ == '__main__': cli() diff --git a/swh/loader/svn/tasks.py b/swh/loader/svn/tasks.py index 7193b15..660fa12 100644 --- a/swh/loader/svn/tasks.py +++ b/swh/loader/svn/tasks.py @@ -1,63 +1,51 @@ -# Copyright (C) 2015-2016 The Software Heritage developers +# Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import shutil - -from os.path import basename - from swh.scheduler.task import Task -from .loader import SWHSvnLoader -from . import utils +from .loader import SWHSvnLoader, SWHSvnLoaderFromDumpArchive class LoadSWHSvnRepositoryTsk(Task): """Import one svn repository to Software Heritage. """ task_queue = 'swh_loader_svn' def run(self, *args, **kwargs): """Import a svn repository with swh policy. Args: args: ordered arguments (expected None) kwargs: Dictionary with the following expected keys: - svn_url: (mandatory) svn's repository url - destination_path: (mandatory) root directory to locally retrieve svn's data - swh_revision: (optional) extra SWH revision hex to start from. cf. swh.loader.svn.SvnLoader.process docstring """ - SWHSvnLoader().load(*args, **kwargs) + loader = SWHSvnLoader() + loader.log = self.log + loader.load(*args, **kwargs) class MountAndLoadSvnRepositoryTsk(Task): task_queue = 'swh_loader_svn_mount_and_load' def run(self, archive_path, origin_url=None, visit_date=None): """1. Mount an svn dump from archive as a local svn repository. 2. Load it through the svn loader. 3. Clean up mounted svn repository archive. + """ - temp_dir = None - try: - self.log.info('Archive to mount and load %s' % archive_path) - temp_dir, repo_path = utils.init_svn_repo_from_archive_dump( - archive_path) - self.log.debug('Mounted svn repository to %s' % repo_path) - SWHSvnLoader().load(svn_url='file://%s' % repo_path, - origin_url=origin_url, - visit_date=visit_date, - destination_path=None) - except Exception as e: - raise e - finally: - if temp_dir: - self.log.debug('Clean up temp directory %s for project %s' % ( - temp_dir, basename(repo_path))) - shutil.rmtree(temp_dir) + + loader = SWHSvnLoaderFromDumpArchive(archive_path) + loader.log = self.log + loader.load(svn_url='file://%s' % loader.repo_path, + origin_url=origin_url, + visit_date=visit_date, + destination_path=None) diff --git a/version.txt b/version.txt index 50caf36..26249a9 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.23-0-g8b3a85c \ No newline at end of file +v0.0.24-0-gffae957 \ No newline at end of file