diff --git a/PKG-INFO b/PKG-INFO index ae1d04a..2ed5e31 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.loader.svn -Version: 0.0.26 +Version: 0.0.27 Summary: Software Heritage Loader SVN Home-page: https://forge.softwareheritage.org/diffusion/DLDSVN Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/bin/swh-hashtree b/bin/swh-hashtree index bcc691b..ad38927 100755 --- a/bin/swh-hashtree +++ b/bin/swh-hashtree @@ -1,34 +1,34 @@ #!/usr/bin/env python3 # Use sample: # swh-hashtree --path . --ignore '.svn' --ignore '.git-svn' \ # --ignore-empty-folders # 38f8d2c3a951f6b94007896d0981077e48bbd702 import click -from swh.core import hashutil +from swh.model import hashutil from swh.loader.svn.utils import hashtree @click.command() @click.option('--path', default='.', help='Optional path to hash.') @click.option('--ignore-empty-folder', is_flag=True, default=False, help='Ignore empty folder.') @click.option('--ignore', multiple=True, help='Ignore pattern.') def main(path, ignore_empty_folder=False, ignore=None): try: h = hashtree(path, ignore_empty_folder, ignore) except Exception as e: print(e) return else: if h: print(hashutil.hash_to_hex(h['sha1_git'])) if __name__ == '__main__': main() diff --git a/debian/control b/debian/control index 7ff5e57..390358d 100644 --- a/debian/control +++ b/debian/control @@ -1,35 +1,35 @@ Source: swh-loader-svn Maintainer: Software Heritage developers Section: python Priority: optional Build-Depends: debhelper (>= 9), dh-python, python3-all, python3-click, python3-dateutil, python3-nose, python3-retrying, python3-setuptools, python3-subvertpy (>= 0.9.4~), python3-swh.core (>= 0.0.19~), - python3-swh.loader.core (>= 0.0.14~), + python3-swh.loader.core (>= 0.0.15~), python3-swh.model (>= 0.0.15~), python3-swh.scheduler (>= 0.0.14~), python3-swh.storage (>= 0.0.83~), python3-vcversioner Standards-Version: 3.9.6 Homepage: https://forge.softwareheritage.org/diffusion/DLDSVN/ Package: python3-swh.loader.svn Architecture: all Depends: gzip, python3-swh.core (>= 0.0.19~), - python3-swh.loader.core (>= 0.0.14~), + python3-swh.loader.core (>= 0.0.15~), python3-swh.model (>= 0.0.15~), python3-swh.scheduler (>= 0.0.14~), python3-swh.storage (>= 0.0.83~), subversion, ${misc:Depends}, ${python3:Depends} Description: Software Heritage Loader Svn Module in charge of loading svn repositories into swh storage. diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..58a761e --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,3 @@ +_build/ +apidoc/ +*-stamp diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..c30c50a --- /dev/null +++ b/docs/Makefile @@ -0,0 +1 @@ +include ../../swh-docs/Makefile.sphinx diff --git a/docs/_static/.placeholder b/docs/_static/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/docs/_templates/.placeholder b/docs/_templates/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..190deb7 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1 @@ +from swh.docs.sphinx.conf import * # NoQA diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..8b64117 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,15 @@ +Software Heritage - Development Documentation +============================================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/requirements-swh.txt b/requirements-swh.txt index e4ff45d..679698d 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,5 +1,5 @@ swh.core >= 0.0.19 swh.storage >= 0.0.83 swh.model >= 0.0.15 swh.scheduler >= 0.0.14 -swh.loader.core >= 0.0.14 +swh.loader.core >= 0.0.15 diff --git a/swh.loader.svn.egg-info/PKG-INFO b/swh.loader.svn.egg-info/PKG-INFO index ae1d04a..2ed5e31 100644 --- a/swh.loader.svn.egg-info/PKG-INFO +++ b/swh.loader.svn.egg-info/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.loader.svn -Version: 0.0.26 +Version: 0.0.27 Summary: Software Heritage Loader SVN Home-page: https://forge.softwareheritage.org/diffusion/DLDSVN Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/swh.loader.svn.egg-info/SOURCES.txt b/swh.loader.svn.egg-info/SOURCES.txt index c576d79..0862885 100644 --- a/swh.loader.svn.egg-info/SOURCES.txt +++ b/swh.loader.svn.egg-info/SOURCES.txt @@ -1,45 +1,51 @@ .gitignore AUTHORS LICENSE MANIFEST.in Makefile README README-dev.org errors.org requirements-swh.txt requirements.txt setup.py svn-lib-client-analysis.org version.txt bin/init-svn-repository.sh bin/swh-hashtree bin/swh-svn debian/changelog debian/compat debian/control debian/copyright debian/rules debian/source/format +docs/.gitignore +docs/Makefile docs/comparison-git-svn-swh-svn.org +docs/conf.py +docs/index.rst docs/swh-loader-svn.txt +docs/_static/.placeholder +docs/_templates/.placeholder install/install-pysvn.sh install/install-subvertpy.sh resources/svn.ini swh.loader.svn.egg-info/PKG-INFO swh.loader.svn.egg-info/SOURCES.txt swh.loader.svn.egg-info/dependency_links.txt swh.loader.svn.egg-info/requires.txt swh.loader.svn.egg-info/top_level.txt swh/loader/svn/__init__.py swh/loader/svn/converters.py swh/loader/svn/loader.py swh/loader/svn/producer.py swh/loader/svn/ra.py swh/loader/svn/svn.py swh/loader/svn/tasks.py swh/loader/svn/utils.py swh/loader/svn/tests/test_base.py swh/loader/svn/tests/test_converters.py swh/loader/svn/tests/test_loader.org swh/loader/svn/tests/test_loader.py swh/loader/svn/tests/test_utils.py \ No newline at end of file diff --git a/swh.loader.svn.egg-info/requires.txt b/swh.loader.svn.egg-info/requires.txt index 5be546d..7d174f8 100644 --- a/swh.loader.svn.egg-info/requires.txt +++ b/swh.loader.svn.egg-info/requires.txt @@ -1,10 +1,10 @@ click python-dateutil retrying subvertpy>=0.9.4 swh.core>=0.0.19 -swh.loader.core>=0.0.14 +swh.loader.core>=0.0.15 swh.model>=0.0.15 swh.scheduler>=0.0.14 swh.storage>=0.0.83 vcversioner diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py index 5745c4c..d440849 100644 --- a/swh/loader/svn/loader.py +++ b/swh/loader/svn/loader.py @@ -1,502 +1,508 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Loader in charge of injecting either new or existing svn mirrors to swh-storage. """ import abc import os import shutil from swh.core import utils from swh.model import git, hashutil from swh.model.git import GitType from swh.loader.core.loader import SWHLoader from . import svn, converters from .utils import hashtree, init_svn_repo_from_archive_dump class SvnLoaderEventful(ValueError): """A wrapper exception to transit the swh_revision onto which the loading failed. """ def __init__(self, e, swh_revision): super().__init__(e) self.swh_revision = swh_revision class SvnLoaderUneventful(ValueError): - pass + def __init__(self, e, *args): + super().__init__(e, *args) class SvnLoaderHistoryAltered(ValueError): - pass + def __init__(self, e, *args): + super().__init__(e, *args) class BaseSvnLoader(SWHLoader, metaclass=abc.ABCMeta): """Base Svn loader to load one svn repository according to specific policies (only swh one now). - The main entry point of this is (no need to override it) - - def load(self, origin_visit, last_known_swh_revision=None): pass + The main entry point of this is (no need to override it):: - Inherit this class and then override the following functions: - - def build_swh_revision(self, rev, commit, dir_id, parents) - This is in charge of converting an svn revision to a compliant - swh revision + def load(self, origin_visit, last_known_swh_revision=None): pass - - def process_repository(self) - This is in charge of processing the actual svn repository and - store the result to swh storage. + Inherit this class and then override the following functions:: + + def build_swh_revision(self, rev, commit, dir_id, parents): + # This is in charge of converting an svn revision to a compliant + # swh revision + + def process_repository(self): + # This is in charge of processing the actual svn repository and + # store the result to swh storage. """ CONFIG_BASE_FILENAME = 'loader/svn' ADDITIONAL_CONFIG = { 'check_revision': ('int', 1000), } def __init__(self): super().__init__(logging_class='swh.loader.svn.SvnLoader') self.check_revision = self.config['check_revision'] @abc.abstractmethod def swh_revision_hash_tree_at_svn_revision(self, revision): """Compute and return the hash tree at a given svn revision. Args: rev (int): the svn revision we want to check Returns: The hash tree directory as bytes. """ pass @abc.abstractmethod def get_svn_repo(self, svn_url, destination_path, origin): """Instantiates the needed svnrepo collaborator to permit reading svn repository. Args: svn_url: the svn repository url to read from destination_path: the local path on disk to compute data origin: the corresponding origin Returns: - Instance of swh.loader.svn.svn clients + Instance of :mod:`swh.loader.svn.svn` clients """ raise NotImplementedError @abc.abstractmethod def build_swh_revision(self, rev, commit, dir_id, parents): """Convert an svn revision to an swh one according to the loader's policy (git-svn or swh). Args: rev: the svn revision number - commit: dictionary with keys 'author_name', 'author_date', 'rev', - 'message' + commit: dictionary with keys: author\_name, author\_date, rev, + message dir_id: the hash tree computation parents: the revision's parents Returns: The swh revision """ raise NotImplementedError @abc.abstractmethod def process_repository(self, origin_visit, last_known_swh_revision=None): """The main idea of this function is to: + - iterate over the svn commit logs - extract the svn commit log metadata - - compute the hashes from the current directory down to the - file + - compute the hashes from the current directory down to the file - compute the equivalent swh revision - send all those objects for storage - - create an swh occurrence pointing to the last swh revision - seen + - create an swh occurrence pointing to the last swh revision seen - send that occurrence for storage in swh-storage. """ raise NotImplementedError def process_svn_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process revisions from revision_start to revision_end and send to swh for storage. At each svn revision, checkout the repository, compute the tree hash and blobs and send for swh storage to store. Then computes and yields the swh revision. Note that at every self.check_revision, an svn export is done and a hash tree is computed to check that no divergence occurred. Yields: swh revision as a dictionary with keys, sha1_git, sha1, etc... """ gen_revs = svnrepo.swh_hash_data_per_revision( revision_start, revision_end) swh_revision = None count = 0 for rev, nextrev, commit, objects_per_path in gen_revs: count += 1 # Send the associated contents/directories self.maybe_load_contents( git.objects_per_type(GitType.BLOB, objects_per_path)) self.maybe_load_directories( git.objects_per_type(GitType.TREE, objects_per_path)) # compute the fs tree's checksums dir_id = objects_per_path[b'']['checksums']['sha1_git'] swh_revision = self.build_swh_revision( rev, commit, dir_id, revision_parents[rev]) swh_revision['id'] = git.compute_revision_sha1_git( swh_revision) self.log.debug('rev: %s, swhrev: %s, dir: %s' % ( rev, hashutil.hash_to_hex(swh_revision['id']), hashutil.hash_to_hex(dir_id))) if (count % self.check_revision) == 0: # hash computation check self.log.info('Checking hash computations on revision %s...' % rev) checked_dir_id = self.swh_revision_hash_tree_at_svn_revision( rev) if checked_dir_id != dir_id: err = 'Hash tree computation divergence detected (%s != %s), stopping!' % ( # noqa hashutil.hash_to_hex(dir_id), hashutil.hash_to_hex(checked_dir_id) ) raise ValueError(err) if nextrev: revision_parents[nextrev] = [swh_revision['id']] yield swh_revision def process_swh_revisions(self, svnrepo, revision_start, revision_end, revision_parents): """Process and store revision to swh (sent by blocks of - 'revision_packet_size') + revision_packet_size) - Returns: - The latest revision stored. + Returns: + The latest revision stored. """ try: swh_revision_gen = self.process_svn_revisions(svnrepo, revision_start, revision_end, revision_parents) revs = [] for revisions in utils.grouper( swh_revision_gen, self.config['revision_packet_size']): revs = list(revisions) self.log.info('Processed %s revisions: [%s, ...]' % ( len(revs), hashutil.hash_to_hex(revs[0]['id']))) self.maybe_load_revisions(revs) except Exception as e: if revs: # flush remaining revisions self.maybe_load_revisions(revs) # Take the last one as the last known revisions known_swh_rev = revs[-1] # Then notify something is wrong, and we stopped at that rev. raise SvnLoaderEventful(e, swh_revision={ 'id': known_swh_rev['id'], }) raise e return revs[-1] def process_swh_occurrence(self, revision, origin_visit): """Process and load the occurrence pointing to the latest revision. """ occ = converters.build_swh_occurrence( revision['id'], origin_visit['origin'], origin_visit['visit']) self.log.debug('occ: %s' % occ) self.maybe_load_occurrences([occ]) def prepare(self, *args, **kwargs): + self.args = args + destination_path = kwargs['destination_path'] # local svn url svn_url = kwargs['svn_url'] origin_url = kwargs.get('origin_url') self.visit_date = kwargs.get('visit_date') origin = { 'url': origin_url if origin_url else svn_url, 'type': 'svn', } self.origin_id = self.send_origin(origin) origin['id'] = self.origin_id self.origin = origin if 'swh_revision' in kwargs: self.last_known_swh_revision = hashutil.hash_to_bytes( kwargs['swh_revision']) else: self.last_known_swh_revision = None self.svnrepo = self.get_svn_repo(svn_url, destination_path, origin) self.fetch_history_id = self.open_fetch_history() def get_origin(self): """Retrieve the origin we are working with. """ return self.origin # set in prepare method def fetch_data(self): """We need to fetch and stream the data to store directly. So - fetch_data do actually nothing. The method `store_data` below - is in charge to do everything, fetch and store. + fetch_data do actually nothing. The method ``store_data`` below is in + charge to do everything, fetch and store. """ pass def store_data(self): """We need to fetch and stream the data to store directly because there is too much data and state changes. Everything is intertwined together (We receive patch and apply on disk and compute at the hashes at the same time) So every data to fetch and store is done here. - Requisite: origin_visit and last_known_swh_revision must have - been set in the prepare method. + Note: + origin_visit and last_known_swh_revision must have been set in the + prepare method. """ origin_visit = {'origin': self.origin_id, 'visit': self.visit} try: latest_rev = self.process_repository(origin_visit, self.last_known_swh_revision) except SvnLoaderEventful as e: self.log.error('Eventful partial visit. Detail: %s' % e) latest_rev = e.swh_revision self.process_swh_occurrence(latest_rev, origin_visit) raise except (SvnLoaderHistoryAltered, SvnLoaderUneventful) as e: self.log.error('Uneventful visit. Detail: %s' % e) raise except Exception as e: raise else: self.process_swh_occurrence(latest_rev, origin_visit) class SWHSvnLoader(BaseSvnLoader): """Swh svn loader is the main implementation destined for production. + This implementation is able to deal with update on known svn repository. Default policy: Keep data as close as possible from the original svn data. We only add information that are needed for update or continuing from last known revision (svn revision and svn repository's uuid). """ - def __init__(self): - super().__init__() - def cleanup(self): """Clean after oneself. This is in charge to flush the remaining data to write in swh storage. And to clean up the svn repository's working representation on disk. """ self.svnrepo.clean_fs() def swh_revision_hash_tree_at_svn_revision(self, revision): """Compute a given hash tree at specific revision. """ local_dirname, local_url = self.svnrepo.export_temporary(revision) h = hashtree(local_url)['sha1_git'] self.svnrepo.clean_fs(local_dirname) return h def get_svn_repo(self, svn_url, destination_path, origin): return svn.SWHSvnRepo( svn_url, origin['id'], self.storage, destination_path=destination_path) def swh_previous_revision(self, prev_swh_revision=None): """Retrieve swh's previous revision if any. """ return self.svnrepo.swh_previous_revision(prev_swh_revision) def check_history_not_altered(self, svnrepo, revision_start, swh_rev): """Given a svn repository, check if the history was not tampered with. """ revision_id = swh_rev['id'] parents = swh_rev['parents'] hash_data_per_revs = svnrepo.swh_hash_data_at_revision(revision_start) rev = revision_start rev, _, commit, objects_per_path = list(hash_data_per_revs)[0] dir_id = objects_per_path[b'']['checksums']['sha1_git'] swh_revision = self.build_swh_revision(rev, commit, dir_id, parents) swh_revision_id = git.compute_revision_sha1_git(swh_revision) return swh_revision_id == revision_id def build_swh_revision(self, rev, commit, dir_id, parents): """Build the swh revision dictionary. This adds: - - the 'synthetic' flag to true - - the 'extra_headers' containing the repository's uuid and the + + - the `'synthetic`' flag to true + - the '`extra_headers`' containing the repository's uuid and the svn revision number. Args: rev: the svn revision commit: the commit metadata dir_id: the upper tree's hash identifier parents: the parents' identifiers Returns: The swh revision corresponding to the svn revision. """ return converters.build_swh_revision(rev, commit, self.svnrepo.uuid, dir_id, parents) def init_from(self, partial_swh_revision, previous_swh_revision): """Function to determine from where to start from. Args: - - partial_swh_revision: A known revision from which - the previous loading did not finish. - - known_previous_revision: A known revision from which the - previous loading did finish. + partial_swh_revision: A known revision from which + the previous loading did not finish. + known_previous_revision: A known revision from which the + previous loading did finish. Returns: The revision from which to start or None if nothing (fresh start). """ if partial_swh_revision and not previous_swh_revision: return partial_swh_revision if not partial_swh_revision and previous_swh_revision: return previous_swh_revision if partial_swh_revision and previous_swh_revision: # will determine from which to start from extra_headers1 = dict( partial_swh_revision['metadata']['extra_headers']) extra_headers2 = dict( previous_swh_revision['metadata']['extra_headers']) rev_start1 = int(extra_headers1['svn_revision']) rev_start2 = int(extra_headers2['svn_revision']) if rev_start1 <= rev_start2: return previous_swh_revision return partial_swh_revision return None def process_repository(self, origin_visit, last_known_swh_revision=None): svnrepo = self.svnrepo # default configuration revision_start = 1 revision_parents = { revision_start: [] } # Check if we already know a previous revision for that origin swh_rev = self.swh_previous_revision() # Determine from which known revision to start swh_rev = self.init_from(last_known_swh_revision, previous_swh_revision=swh_rev) if swh_rev: # Yes, we do. Try and update it. extra_headers = dict(swh_rev['metadata']['extra_headers']) revision_start = int(extra_headers['svn_revision']) revision_parents = { revision_start: swh_rev['parents'], } self.log.debug('svn export --ignore-keywords %s@%s' % ( svnrepo.remote_url, revision_start)) if swh_rev and not self.check_history_not_altered( svnrepo, revision_start, swh_rev): msg = 'History of svn %s@%s history modified. Skipping...' % ( svnrepo.remote_url, revision_start) self.log.warn(msg) - raise SvnLoaderHistoryAltered + raise SvnLoaderHistoryAltered(msg, *self.args) else: # now we know history is ok, we start at next revision revision_start = revision_start + 1 # and the parent become the latest know revision for # that repository revision_parents[revision_start] = [swh_rev['id']] revision_end = svnrepo.head_revision() self.log.info('[revision_start-revision_end]: [%s-%s]' % ( revision_start, revision_end)) if revision_start > revision_end and revision_start is not 1: - self.log.info('%s@%s already injected.' % ( - svnrepo.remote_url, revision_end)) - raise SvnLoaderUneventful + msg = '%s@%s already injected.' % (svnrepo.remote_url, + revision_end) + self.log.info(msg) + raise SvnLoaderUneventful(msg, *self.args) self.log.info('Processing %s.' % svnrepo) # process and store revision to swh (sent by by blocks of # 'revision_packet_size') return self.process_swh_revisions( svnrepo, revision_start, revision_end, revision_parents) class SWHSvnLoaderFromDumpArchive(SWHSvnLoader): """Load a svn repository from an archive (containing a dump). """ def __init__(self, archive_path): super().__init__() self.log.info('Archive to mount and load %s' % archive_path) self.temp_dir, self.repo_path = init_svn_repo_from_archive_dump( archive_path) def cleanup(self): super().cleanup() if self.temp_dir and os.path.exists(self.temp_dir): self.log.debug('Clean up temp directory %s for project %s' % ( self.temp_dir, os.path.basename(self.repo_path))) shutil.rmtree(self.temp_dir) diff --git a/swh/loader/svn/ra.py b/swh/loader/svn/ra.py index 044ae50..2c0fb8b 100644 --- a/swh/loader/svn/ra.py +++ b/swh/loader/svn/ra.py @@ -1,500 +1,507 @@ # Copyright (C) 2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Remote Access client to svn server. """ import click import os import shutil import tempfile from subvertpy import delta, properties from subvertpy.ra import RemoteAccess, Auth, get_username_provider from swh.model import git, hashutil def compute_svn_link_metadata(linkpath, filetype, data): """Given a svn linkpath (raw file with format 'link '), compute the git metadata. Args: linkpath: absolute pathname of the svn link filetype: the file's type according to svn, should only be 'link' data: the link's content a.k.a the link's source file Returns: - Dictionary of values: - - data: link's content - - length: link's content length - - name: basename of the link - - perms: git permission for link - - type: git type for link - - path: absolute path to the link on filesystem + dict: a dictionary with the following keys: + + - data: link's content + - length: link's content length + - name: basename of the link + - perms: git permission for link + - type: git type for link + - path: absolute path to the link on filesystem Raises: ValueError if the filetype does not match 'link'. """ if filetype != b'link': raise ValueError( 'Do not deal with other type (%s) than link.' % ( linkpath, type)) link_metadata = hashutil.hash_data(data) link_metadata.update({ 'data': data, 'length': len(data), 'name': os.path.basename(linkpath), 'perms': git.GitPerm.LINK, 'type': git.GitType.BLOB, 'path': linkpath }) return link_metadata def apply_txdelta_handler(sbuf, target_stream): """Return a function that can be called repeatedly with txdelta windows. When done, closes the target_stream. Adapted from subvertpy.delta.apply_txdelta_handler to close the stream when done. - Args + Args: sbuf: Source buffer target_stream: Target stream to write to. - Returns - Function to be called to apply txdelta windows + Returns: + Function to be called to apply txdelta windows """ def apply_window(window): if window is None: target_stream.close() return # Last call patch = delta.apply_txdelta_window(sbuf, window) target_stream.write(patch) return apply_window class SWHFileEditor: """File Editor in charge of updating file on disk and memory objects. """ __slots__ = ['objects', 'path', 'fullpath', 'executable', 'link'] def __init__(self, objects, rootpath, path): self.objects = objects self.path = path # default value: 0, 1: set the flag, 2: remove the exec flag self.executable = 0 self.link = None self.fullpath = os.path.join(rootpath, path) def change_prop(self, key, value): if key == properties.PROP_EXECUTABLE: if value is None: # bit flip off self.executable = 2 else: self.executable = 1 elif key == properties.PROP_SPECIAL: self.link = True def __make_symlink(self): """Convert the svnlink to a symlink on disk. This function expects self.fullpath to be a svn link. Return: - The svnlink's data tuple: - - type (should be only 'link') - - + tuple: The svnlink's data tuple: + + - type (should be only 'link') + - """ with open(self.fullpath, 'rb') as f: filetype, src = f.read().split(b' ') os.remove(self.fullpath) os.symlink(src=src, dst=self.fullpath) return filetype, src def __make_svnlink(self): """Convert the symlink to a svnlink on disk. Return: - The symlink's svnlink data (b'type ') + The symlink's svnlink data (``b'type '``) """ # we replace the symlink by a svnlink src = os.readlink(self.fullpath) os.remove(self.fullpath) # to be able to patch the file sbuf = b'link ' + src with open(self.fullpath, 'wb') as f: f.write(sbuf) return sbuf def apply_textdelta(self, base_checksum): if os.path.lexists(self.fullpath): if os.path.islink(self.fullpath): sbuf = self.__make_svnlink() self.link = True else: with open(self.fullpath, 'rb') as f: sbuf = f.read() else: sbuf = b'' t = open(self.fullpath, 'wb') return apply_txdelta_handler(sbuf, target_stream=t) def close(self): """When done with the file, this is called. + So the file exists and is updated, we can: + - adapt accordingly its execution flag if any - compute the objects' checksums """ if self.link: filetype, source_link = self.__make_symlink() self.objects[self.path] = { 'checksums': compute_svn_link_metadata(self.fullpath, filetype=filetype, data=source_link) } return if self.executable == 1: os.chmod(self.fullpath, 0o755) elif self.executable == 2: os.chmod(self.fullpath, 0o644) # And now compute file's checksums self.objects[self.path] = { 'checksums': git.compute_blob_metadata(self.fullpath) } def default_dictionary(): """Default dictionary. """ return dict(checksums=None, children=set()) class BaseDirSWHEditor: """Base class implementation of dir editor. - cf. SWHDirEditor for an implementation that hashes every directory - encountered. + see :class:`SWHDirEditor` for an implementation that hashes every + directory encountered. + + cf. :class:`SWHDirEditorNoEmptyFolder` for an implementation that + deletes empty folder + + Instantiate a new class inheriting from this class and define the following + functions:: - cf. SWHDirEditorNoEmptyFolder for an implementation that deletes - empty folder + def update_checksum(self): + # Compute the checksums at current state - Instantiate a new class inheriting from this class and define the - following function: + def open_directory(self, *args): + # Update an existing folder. - - def update_checksum(self): - Compute the checksums at current state - - def open_directory(self, *args): - Update an existing folder. - - def add_directory(self, *args): - Add a new one. + def add_directory(self, *args): + # Add a new one. """ __slots__ = ['objects', 'rootpath', 'path'] def __init__(self, objects, rootpath, path): self.objects = objects self.rootpath = rootpath self.path = path # build directory on init os.makedirs(os.path.join(rootpath, path), exist_ok=True) def add_child(self, path): """Add a children path to the actual objects for the current directory seen as the parent. Args: path: The child to add """ d = self.objects.get(self.path, default_dictionary()) d['children'].add(path) self.objects[self.path] = d def remove_child(self, path): """Remove a path from the current objects. The path can be resolved as link, file or directory. This function takes also care of removing the link between the child and the parent. Args: path: to remove from the current objects. """ entry_removed = self.objects.pop(path, None) fpath = os.path.join(self.rootpath, path) if entry_removed: if 'children' in entry_removed: # dir for child_path in entry_removed['children']: self.remove_child(child_path) parent = os.path.dirname(path) if parent and parent in self.objects: self.objects[parent]['children'].discard(path) if os.path.lexists(fpath): # we want to catch broken symlink too if os.path.isfile(fpath): os.remove(fpath) elif os.path.islink(fpath): os.remove(fpath) else: shutil.rmtree(fpath) def update_checksum(self): raise NotImplementedError('This should be implemented.') def open_directory(self, *args): raise NotImplementedError('This should be implemented.') def add_directory(self, *args): raise NotImplementedError('This should be implemented.') def open_file(self, *args): """Updating existing file. """ path = args[0].encode('utf-8') self.add_child(path) return SWHFileEditor(self.objects, rootpath=self.rootpath, path=path) def add_file(self, path, copyfrom_path=None, copyfrom_rev=-1): """Creating a new file. """ path = path.encode('utf-8') self.add_child(path) return SWHFileEditor(self.objects, rootpath=self.rootpath, path=path) def change_prop(self, key, value): """Change property callback on directory. """ if key == properties.PROP_EXTERNALS: raise ValueError( "Property '%s' detected. Not implemented yet." % key) def delete_entry(self, path, revision): """Remove a path. """ self.remove_child(path.encode('utf-8')) def close(self): """Function called when we finish walking a repository. """ self.update_checksum() class SWHDirEditor(BaseDirSWHEditor): """Directory Editor in charge of updating directory hashes computation. This implementation includes empty folder in the hash computation. """ def update_checksum(self): """Update the root path self.path's checksums according to the children's objects. This function is expected to be called when the folder has been completely 'walked'. """ d = self.objects.get(self.path, default_dictionary()) # Retrieve the list of the current folder's children objects ls_hashes = list(git.children_hashes(d['children'], objects=self.objects)) d['checksums'] = git._compute_tree_metadata(self.path, ls_hashes) self.objects[self.path] = d def open_directory(self, *args): """Updating existing directory. """ path = args[0].encode('utf-8') self.add_child(path) return SWHDirEditor(self.objects, self.rootpath, path=path) def add_directory(self, path, copyfrom_path=None, copyfrom_rev=-1): """Adding a new directory. """ path = path.encode('utf-8') self.add_child(path) return SWHDirEditor(self.objects, rootpath=self.rootpath, path=path) class BaseSWHEditor: """SWH Base class editor in charge of receiving events. """ def __init__(self, rootpath, objects): self.rootpath = rootpath self.objects = objects def set_target_revision(self, revnum): pass def abort(self): pass def close(self): pass def open_root(self, base_revnum): raise NotImplementedError('Instantiate an swh dir editor of your ' ' choice depending of the hash computation ' ' policy you want') class SWHEditor(BaseSWHEditor): """SWH Editor in charge of replaying svn events and computing objects along. This implementation accounts for empty folder during hash computations. """ def open_root(self, base_revnum): return SWHDirEditor(self.objects, rootpath=self.rootpath, path=b'') class BaseSWHReplay: """Base replay class. Their role is to compute objects for a particular revision. This class is intended to be inherited to: - - initialize the editor (global loading policy depends on this editor) + - initialize the editor (global loading policy depends on this editor) - override the compute_hashes function in charge of computing - hashes between rev and rev+1 + hashes between rev and rev+1 - cf. SWHReplayNoEmptyFolder and SWHReplay for instanciated classes. + cf. :class:`SWHReplayNoEmptyFolder` and :class:`SWHReplay` for + instanciated classes. """ def replay(self, rev): """Replay svn actions between rev and rev+1. - This method updates in place the self.editor.objects's reference. - This also updates in place the filesystem. + This method updates in place the self.editor.objects's reference. This + also updates in place the filesystem. Returns: The updated objects """ self.conn.replay(rev, rev+1, self.editor) return self.editor.objects def compute_hashes(self, rev): """Compute hashes at revisions rev. Expects the objects to be at previous revision's objects. Args: rev: The revision to start the replay from. Returns: - The updated objects between rev and rev+1. - Beware that this mutates the filesystem at rootpath accordingly. + The updated objects between rev and rev+1. Beware that this mutates + the filesystem at rootpath accordingly. """ raise NotImplementedError('This should be overridden by subclass') class SWHReplay(BaseSWHReplay): """Replay class. All folders and files are considered for hash computations. """ def __init__(self, conn, rootpath, objects=None): self.conn = conn self.rootpath = rootpath self.editor = SWHEditor(rootpath=rootpath, objects=objects if objects else {}) def compute_hashes(self, rev): """Compute hashes at revisions rev. Expects the state to be at previous revision's objects. Args: rev: The revision to start the replay from. Returns: - The updated objects between rev and rev+1. - Beware that this mutates the filesystem at rootpath accordingly. + The updated objects between rev and rev+1. Beware that this + mutates the filesystem at rootpath accordingly. """ return self.replay(rev) @click.command() @click.option('--local-url', default='/tmp', help="local svn working copy") @click.option('--svn-url', default='file:///home/storage/svn/repos/pkg-fox', help="svn repository's url.") @click.option('--revision-start', default=1, type=click.INT, help="svn repository's starting revision.") @click.option('--revision-end', default=-1, type=click.INT, help="svn repository's ending revision.") @click.option('--debug/--nodebug', default=True, help="Indicates if the server should run in debug mode.") @click.option('--cleanup/--nocleanup', default=True, help="Indicates whether to cleanup disk when done or not.") def main(local_url, svn_url, revision_start, revision_end, debug, cleanup): """Script to present how to use SWHReplay class. """ conn = RemoteAccess(svn_url.encode('utf-8'), auth=Auth([get_username_provider()])) os.makedirs(local_url, exist_ok=True) rootpath = tempfile.mkdtemp(prefix=local_url, suffix='-'+os.path.basename(svn_url)) rootpath = rootpath.encode('utf-8') # Do not go beyond the repository's latest revision revision_end_max = conn.get_latest_revnum() if revision_end == -1: revision_end = revision_end_max revision_end = min(revision_end, revision_end_max) try: replay = SWHReplay(conn, rootpath) for rev in range(revision_start, revision_end+1): objects = replay.compute_hashes(rev) print('r%s %s' % (rev, hashutil.hash_to_hex( objects[b'']['checksums']['sha1_git']))) if debug: print('%s' % rootpath.decode('utf-8')) finally: if cleanup: if os.path.exists(rootpath): shutil.rmtree(rootpath) if __name__ == '__main__': main() diff --git a/swh/loader/svn/svn.py b/swh/loader/svn/svn.py index ef45e9c..365b851 100644 --- a/swh/loader/svn/svn.py +++ b/swh/loader/svn/svn.py @@ -1,370 +1,376 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """SVN client in charge of iterating over svn logs and yield commit representations including the hash tree/content computations per svn commit. """ import os import tempfile import shutil from subvertpy.ra import RemoteAccess, Auth, get_username_provider from subvertpy import client, properties from swh.model import git from . import ra, utils, converters # When log message contains empty data DEFAULT_AUTHOR_MESSAGE = '' class SvnRepoException(ValueError): def __init__(self, svnrepo, e): super().__init__(e) self.svnrepo = svnrepo class BaseSvnRepo(): """Base svn repository representation for swh. - To override some of the behavior regarding the message log - properties, you can instantiate a subclass of this class and - override: - - def convert_commit_author(self, author) - - def convert_commit_message(self, msg) - - def convert_commit_date(self, date) + To override some of the behavior regarding the message log properties, you + can instantiate a subclass of this class and override:: - cf. GitSvnSvnRepo, SwhSvnRepo for instanciation example. + def convert_commit_author(self, author) + def convert_commit_message(self, msg) + def convert_commit_date(self, date) + + see :class:`GitSvnSvnRepo`, :class:`SwhSvnRepo` for instanciation + example. """ def __init__(self, remote_url, origin_id, storage, destination_path=None): self.remote_url = remote_url.rstrip('/') self.storage = storage self.origin_id = origin_id if destination_path: os.makedirs(destination_path, exist_ok=True) self.root_dir = destination_path else: self.root_dir = '/tmp' auth = Auth([get_username_provider()]) # one connection for log iteration self.conn_log = RemoteAccess(self.remote_url, auth=auth) # another for replay self.conn = RemoteAccess(self.remote_url, auth=auth) # one client for update operation self.client = client.Client(auth=auth) self.local_dirname = tempfile.mkdtemp( suffix='.swh.loader', prefix='tmp.', dir=self.root_dir) local_name = os.path.basename(self.remote_url) self.local_url = os.path.join(self.local_dirname, local_name).encode( 'utf-8') self.uuid = self.conn.get_uuid().encode('utf-8') def __str__(self): return str({ 'remote_url': self.remote_url, 'local_url': self.local_url, 'uuid': self.uuid, 'swh-origin': self.origin_id }) def head_revision(self): """Retrieve current revision of the repository's working copy. """ return self.conn.get_latest_revnum() def initial_revision(self): """Retrieve the initial revision from which the remote url appeared. Note: This should always be 1 since we won't be dealing with in-depth url. """ return 1 def convert_commit_message(self, msg): """Do something with message (e.g add extra line, etc...) cf. SvnRepo for a simple implementation. Args: msg (str): the commit message to convert. Returns: The transformed message as bytes. """ raise NotImplementedError('Should be overridden by subclass.') def convert_commit_date(self, date): """Convert the message date (e.g, convert into timestamp or whatever makes sense to you.). Args: date (str): the commit date to convert. Returns: The transformed date. """ raise NotImplementedError('Should be overridden by subclass.') def convert_commit_author(self, author): """Convert the commit author (e.g, convert into dict or whatever makes sense to you.). Args: author (str): the commit author to convert. Returns: The transformed author as dict. """ raise NotImplementedError('Should be overridden by subclass.') def __to_entry(self, log_entry): changed_paths, rev, revprops, has_children = log_entry author_date = self.convert_commit_date( revprops.get(properties.PROP_REVISION_DATE)) author = self.convert_commit_author( revprops.get(properties.PROP_REVISION_AUTHOR)) message = self.convert_commit_message( revprops.get(properties.PROP_REVISION_LOG, DEFAULT_AUTHOR_MESSAGE)) return { 'rev': rev, 'author_date': author_date, 'author_name': author, 'message': message, } def logs(self, revision_start, revision_end): """Stream svn logs between revision_start and revision_end by chunks of block_size logs. Yields revision and associated revision information between the revision start and revision_end. Args: revision_start: the svn revision starting bound revision_end: the svn revision ending bound Yields: - tuple of revisions and logs. - revisions: list of revisions in order - logs: Dictionary with key revision number and value the log entry. - The log entry is a dictionary with the following keys: - - author_date: date of the commit - - author_name: name of the author - - message: commit message + tuple: tuple of revisions and logs: + + - revisions: list of revisions in order + - logs: Dictionary with key revision number and value the log + entry. The log entry is a dictionary with the following keys: + + - author_date: date of the commit + - author_name: name of the author + - message: commit message """ for log_entry in self.conn_log.iter_log(paths=None, start=revision_start, end=revision_end, discover_changed_paths=False): yield self.__to_entry(log_entry) def export(self, revision): """Export the repository to a given version. """ self.client.export(self.remote_url, to=self.local_url.decode('utf-8'), rev=revision, ignore_keywords=True) def export_temporary(self, revision): """Export the repository to a given revision in a temporary location. This is up to the caller of this function to clean up the temporary location when done (cf. self.clean_fs method) Args: revision: Revision to export at Returns: The tuple local_dirname the temporary location root folder, local_url where the repository was exported. """ local_dirname = tempfile.mkdtemp( prefix='check-revision-%s.' % revision, dir=self.root_dir) local_name = os.path.basename(self.remote_url) local_url = os.path.join(local_dirname, local_name) self.client.export( self.remote_url, to=local_url, rev=revision, ignore_keywords=True) return local_dirname, local_url def swh_previous_revision(self, previous_swh_revision=None): """Look for possible existing revision in swh. Args: previous_swh_revision: (optional) id of a possible previous swh revision Returns: If previous_swh_revision is not None and do exists, returns the complete instance. Otherwise, check for a possible occurrence and returns the targeted complete revision if it does exists. Otherwise, returns None. """ storage = self.storage # got no previous revision, will check if some occurrence # already exists for that origin if not previous_swh_revision: occ = list(storage.occurrence_get(self.origin_id)) if occ: revision_id = occ[0]['target'] revisions = list(storage.revision_get([revision_id])) if revisions: return revisions[0] else: revs = list(storage.revision_get([previous_swh_revision])) if revs: return revs[0] def swh_hash_data_per_revision(self, start_revision, end_revision): """Compute swh hash data per each revision between start_revision and end_revision. Args: start_revision: starting revision end_revision: ending revision Yields: tuple (rev, nextrev, commit, objects_per_path) - rev: current revision - nextrev: next revision - commit: commit data (author, date, message) for such revision - objects_per_path: dictionary of path, swh hash data with type """ hashes = {} for commit in self.logs(start_revision, end_revision): rev = commit['rev'] hashes = self.swhreplay.compute_hashes(rev) if rev == end_revision: nextrev = None else: nextrev = rev + 1 yield rev, nextrev, commit, hashes def swh_hash_data_at_revision(self, revision): """Compute the hash data at revision. Expected to be used for update only. """ # Update the disk at revision self.export(revision) # Compute the current hashes on disk hashes = git.compute_hashes_from_directory(self.local_url) hashes = utils.convert_hashes_with_relative_path( hashes, rootpath=self.local_url) # Update the replay collaborator with the right state self.swhreplay = ra.SWHReplay( conn=self.conn, rootpath=self.local_url, objects=hashes) # Retrieve the commit information for revision commit = list(self.logs(revision, revision))[0] yield revision, revision + 1, commit, hashes def clean_fs(self, local_dirname=None): """Clean up the local working copy. Args: local_dirname (str): Path to remove recursively if provided. Otherwise, remove the temporary upper root tree used for svn repository loading. """ if local_dirname: shutil.rmtree(local_dirname) else: shutil.rmtree(self.local_dirname) class SWHSvnRepo(BaseSvnRepo): - """This class does exactly as BaseSvnRepo except for: + """Same as :class:`BaseSvnRepo` except for: + - the commit message which is simply encoded - the commit author is left as is. - the commit timestamp is left as is. """ def __init__(self, remote_url, origin_id, storage, destination_path=None): super().__init__(remote_url, origin_id, storage, destination_path=destination_path) self.swhreplay = ra.SWHReplay( conn=self.conn, rootpath=self.local_url) def convert_commit_message(self, msg): """Simply encode the commit message. Args: msg (str): the commit message to convert. Returns: The transformed message as bytes. """ return msg.encode('utf-8') def convert_commit_date(self, date): """Convert the message commit date into a timestamp in swh format. The precision is kept. Args: date (str): the commit date to convert. Returns: The transformed date. """ return converters.svn_date_to_swh_date(date) def convert_commit_author(self, author): """Convert the commit author into an swh person. - The user becomes a dictionary of the form: - { - name: author, - email: '', - fullname: author - } + + The user becomes a dictionary of the form:: + + { + name: author, + email: '', + fullname: author + } Args: author (str): the commit author to convert. Returns: The transformed author as dict. """ return converters.svn_author_to_swh_person(author) diff --git a/swh/loader/svn/tasks.py b/swh/loader/svn/tasks.py index 65009a2..0184bcc 100644 --- a/swh/loader/svn/tasks.py +++ b/swh/loader/svn/tasks.py @@ -1,51 +1,52 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.scheduler.task import Task from .loader import SWHSvnLoader, SWHSvnLoaderFromDumpArchive class LoadSWHSvnRepositoryTsk(Task): """Import one svn repository to Software Heritage. """ task_queue = 'swh_loader_svn' def run_task(self, *args, **kwargs): """Import a svn repository with swh policy. Args: args: ordered arguments (expected None) kwargs: Dictionary with the following expected keys: + - svn_url: (mandatory) svn's repository url - destination_path: (mandatory) root directory to locally retrieve svn's data - swh_revision: (optional) extra SWH revision hex to - start from. cf. swh.loader.svn.SvnLoader.process + start from. see swh.loader.svn.SvnLoader.process docstring """ loader = SWHSvnLoader() loader.log = self.log loader.load(*args, **kwargs) class MountAndLoadSvnRepositoryTsk(Task): task_queue = 'swh_loader_svn_mount_and_load' def run_task(self, archive_path, origin_url=None, visit_date=None): """1. Mount an svn dump from archive as a local svn repository. 2. Load it through the svn loader. 3. Clean up mounted svn repository archive. """ loader = SWHSvnLoaderFromDumpArchive(archive_path) loader.log = self.log loader.load(svn_url='file://%s' % loader.repo_path, origin_url=origin_url, visit_date=visit_date, destination_path=None) diff --git a/swh/loader/svn/tests/test_loader.py b/swh/loader/svn/tests/test_loader.py index b0d0e04..ebf193e 100644 --- a/swh/loader/svn/tests/test_loader.py +++ b/swh/loader/svn/tests/test_loader.py @@ -1,619 +1,621 @@ # Copyright (C) 2016-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from nose.tools import istest from swh.model import hashutil from swh.loader.svn.loader import SWHSvnLoader from swh.loader.svn.loader import SvnLoaderHistoryAltered, SvnLoaderUneventful from test_base import BaseTestSvnLoader # Define loaders with no storage # They'll just accumulate the data in place # Only for testing purposes. class TestSvnLoader: """Mixin class to inhibit the persistence and keep in memory the data sent for storage. cf. SWHSvnLoaderNoStorage """ def __init__(self): super().__init__() # We don't want to persist any result in this test context self.config['send_contents'] = False self.config['send_directories'] = False self.config['send_revisions'] = False self.config['send_releases'] = False self.config['send_occurrences'] = False # Init the state self.all_contents = [] self.all_directories = [] self.all_revisions = [] self.all_releases = [] self.all_occurrences = [] # Check at each svn revision that the hash tree computation # does not diverge self.check_revision = 1 def maybe_load_contents(self, all_contents): self.all_contents.extend(all_contents) def maybe_load_directories(self, all_directories): self.all_directories.extend(all_directories) def maybe_load_revisions(self, all_revisions): self.all_revisions.extend(all_revisions) def maybe_load_releases(self, releases): raise ValueError('If called, the test must break.') def maybe_load_occurrences(self, all_occurrences): self.all_occurrences.extend(all_occurrences) # Override to do nothing at the end def close_failure(self): pass def close_success(self): pass # Override to only prepare the svn repository def prepare(self, *args, **kwargs): self.svnrepo = self.get_svn_repo(*args) class SWHSvnLoaderNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: Load a new svn repository using the swh policy (so no update). """ def swh_previous_revision(self, prev_swh_revision=None): """We do not know this repository so no revision. """ return None class SWHSvnLoaderUpdateNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: Load a known svn repository using the swh policy. We can either: - do nothing since it does not contain any new commit (so no change) - either check its history is not altered and update in consequence by loading the new revision """ def swh_previous_revision(self, prev_swh_revision=None): """Avoid the storage persistence call and return the expected previous revision for that repository. Check the following for explanation about the hashes: - test_loader.org for (swh policy). - cf. SWHSvnLoaderITTest """ return { 'id': hashutil.hash_to_bytes( '4876cb10aec6f708f7466dddf547567b65f6c39c'), 'parents': [hashutil.hash_to_bytes( 'a3a577948fdbda9d1061913b77a1588695eadb41')], 'directory': hashutil.hash_to_bytes( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '6'] ] } } class SWHSvnLoaderUpdateHistoryAlteredNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: Load a known svn repository using the swh policy with its history altered so we do not update it. """ def swh_previous_revision(self, prev_swh_revision=None): """Avoid the storage persistence call and return the expected previous revision for that repository. Check the following for explanation about the hashes: - test_loader.org for (swh policy). - cf. SWHSvnLoaderITTest """ return { # Changed the revision id's hash to simulate history altered 'id': hashutil.hash_to_bytes( 'badbadbadbadf708f7466dddf547567b65f6c39d'), 'parents': [hashutil.hash_to_bytes( 'a3a577948fdbda9d1061913b77a1588695eadb41')], 'directory': hashutil.hash_to_bytes( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', b'6'] ] } } class SWHSvnLoaderNewRepositoryITTest(BaseTestSvnLoader): def setUp(self): super().setUp() self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.origin_visit = { 'origin': self.origin['id'], 'visit': 2, } self.loader = SWHSvnLoaderNoStorage() self.loader.prepare( self.svn_mirror_url, self.destination_path, self.origin) @istest def process_repository(self): """Process a new repository with swh policy should be ok. """ # when self.loader.process_repository(self.origin_visit) # then self.assertEquals(len(self.loader.all_revisions), 6) self.assertEquals(len(self.loader.all_releases), 0) last_revision = '4876cb10aec6f708f7466dddf547567b65f6c39c' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '0d7dd5f751cef8fe17e8024f7d6b0e3aac2cfd71': '669a71cce6c424a81ba42b7dc5d560d32252f0ca', # noqa '95edacc8848369d6fb1608e887d6d2474fd5224f': '008ac97a1118560797c50e3392fa1443acdaa349', # noqa 'fef26ea45a520071711ba2b9d16a2985ee837021': '3780effbe846a26751a95a8c95c511fb72be15b4', # noqa '3f51abf3b3d466571be0855dfa67e094f9ceff1b': 'ffcca9b09c5827a6b8137322d4339c8055c3ee1e', # noqa 'a3a577948fdbda9d1061913b77a1588695eadb41': '7dc52cc04c3b8bd7c085900d60c159f7b846f866', # noqa last_revision: '0deab3023ac59398ae467fc4bff5583008af1ee2', # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) class SWHSvnLoaderUpdateWithNoChangeITTest(BaseTestSvnLoader): def setUp(self): super().setUp() self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.origin_visit = { 'origin': self.origin['id'], 'visit': 3, } self.loader = SWHSvnLoaderUpdateNoStorage() self.loader.prepare( self.svn_mirror_url, self.destination_path, self.origin) @istest def process_repository(self): """Process a known repository with swh policy and no new data should be ok. """ # when with self.assertRaises(SvnLoaderUneventful): + self.loader.args = (self.origin_visit,) self.loader.process_repository(self.origin_visit) # then self.assertEquals(len(self.loader.all_revisions), 0) self.assertEquals(len(self.loader.all_releases), 0) class SWHSvnLoaderUpdateWithHistoryAlteredITTest(BaseTestSvnLoader): def setUp(self): # the svn repository pkg-gourmet has been updated with changes super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.origin_visit = { 'origin': self.origin['id'], 'visit': 4, } self.loader = SWHSvnLoaderUpdateHistoryAlteredNoStorage() self.loader.prepare( self.svn_mirror_url, self.destination_path, self.origin) @istest def process_repository(self): - """Process a known repository with swh policy and history altered should - stop and do nothing. + """Process a known repository with swh policy and history altered + should stop and do nothing. """ # when with self.assertRaises(SvnLoaderHistoryAltered): + self.loader.args = (self.origin_visit,) self.loader.process_repository(self.origin_visit) # then # we got the previous run's last revision (rev 6) # so 2 news + 1 old self.assertEquals(len(self.loader.all_revisions), 0) self.assertEquals(len(self.loader.all_releases), 0) class SWHSvnLoaderUpdateWithChangesITTest(BaseTestSvnLoader): def setUp(self): # the svn repository pkg-gourmet has been updated with changes super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.origin_visit = { 'origin': self.origin['id'], 'visit': 5, } self.loader = SWHSvnLoaderUpdateNoStorage() self.loader.prepare( self.svn_mirror_url, self.destination_path, self.origin) @istest def process_repository(self): """Process a known repository with swh policy and new data should yield new revisions and occurrence. """ # when self.loader.process_repository(self.origin_visit) # then # we got the previous run's last revision (rev 6) # so 2 new self.assertEquals(len(self.loader.all_revisions), 5) self.assertEquals(len(self.loader.all_releases), 0) last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) class SWHSvnLoaderUpdateWithUnfinishedLoadingChangesITTest(BaseTestSvnLoader): def setUp(self): super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.origin_visit = { 'origin': self.origin['id'], 'visit': 6 } self.loader = SWHSvnLoaderNoStorage() self.loader.prepare( self.svn_mirror_url, self.destination_path, self.origin) @istest def process_repository(self): """Process a known repository with swh policy, the previous run did not finish, so this finishes the loading """ previous_unfinished_revision = { 'id': hashutil.hash_to_bytes( '4876cb10aec6f708f7466dddf547567b65f6c39c'), 'parents': [hashutil.hash_to_bytes( 'a3a577948fdbda9d1061913b77a1588695eadb41')], 'directory': hashutil.hash_to_bytes( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '6'] ] } } # when self.loader.process_repository( self.origin_visit, last_known_swh_revision=previous_unfinished_revision) # then # we got the previous run's last revision (rev 6) # so 2 new self.assertEquals(len(self.loader.all_revisions), 5) self.assertEquals(len(self.loader.all_releases), 0) last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) class SWHSvnLoaderUpdateWithUnfinishedLoadingChangesButOccurrenceDoneITTest( BaseTestSvnLoader): def setUp(self): super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.origin_visit = { 'origin': self.origin['id'], 'visit': 9, } self.loader = SWHSvnLoaderUpdateNoStorage() self.loader.prepare( self.svn_mirror_url, self.destination_path, self.origin) @istest def process_repository(self): """known repository, swh policy, unfinished revision is less recent than occurrence, we start from last occurrence. """ previous_unfinished_revision = { 'id': hashutil.hash_to_bytes( 'a3a577948fdbda9d1061913b77a1588695eadb41'), 'parents': [hashutil.hash_to_bytes( '3f51abf3b3d466571be0855dfa67e094f9ceff1b')], 'directory': hashutil.hash_to_bytes( '7dc52cc04c3b8bd7c085900d60c159f7b846f866'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '5'] ] } } # when self.loader.process_repository( self.origin_visit, last_known_swh_revision=previous_unfinished_revision) # then # we got the previous run's last revision (rev 6) # so 2 new self.assertEquals(len(self.loader.all_revisions), 5) self.assertEquals(len(self.loader.all_releases), 0) last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) class SWHSvnLoaderUpdateLessRecentNoStorage(TestSvnLoader, SWHSvnLoader): """An SWHSVNLoader with no persistence. Context: Load a known svn repository using the swh policy. The last occurrence seen is less recent than a previous unfinished crawl. """ def swh_previous_revision(self, prev_swh_revision=None): """Avoid the storage persistence call and return the expected previous revision for that repository. Check the following for explanation about the hashes: - test_loader.org for (swh policy). - cf. SWHSvnLoaderITTest """ return { 'id': hashutil.hash_to_bytes( 'a3a577948fdbda9d1061913b77a1588695eadb41'), 'parents': [hashutil.hash_to_bytes( '3f51abf3b3d466571be0855dfa67e094f9ceff1b')], 'directory': hashutil.hash_to_bytes( '7dc52cc04c3b8bd7c085900d60c159f7b846f866'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '5'] ] } } class SWHSvnLoaderUnfinishedLoadingChangesSinceLastOccurrenceITTest( BaseTestSvnLoader): def setUp(self): super().setUp(archive_name='pkg-gourmet-with-updates.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.origin_visit = { 'origin': self.origin['id'], 'visit': 1, } self.loader = SWHSvnLoaderUpdateLessRecentNoStorage() self.loader.prepare( self.svn_mirror_url, self.destination_path, self.origin) @istest def process_repository(self): """known repository, swh policy, unfinished revision is less recent than occurrence, we start from last occurrence. """ previous_unfinished_revision = { 'id': hashutil.hash_to_bytes( '4876cb10aec6f708f7466dddf547567b65f6c39c'), 'parents': [hashutil.hash_to_bytes( 'a3a577948fdbda9d1061913b77a1588695eadb41')], 'directory': hashutil.hash_to_bytes( '0deab3023ac59398ae467fc4bff5583008af1ee2'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '6'] ] } } # when self.loader.process_repository( self.origin_visit, last_known_swh_revision=previous_unfinished_revision) # then # we got the previous run's last revision (rev 6) # so 2 new self.assertEquals(len(self.loader.all_revisions), 5) self.assertEquals(len(self.loader.all_releases), 0) last_revision = '171dc35522bfd17dda4e90a542a0377fb2fc707a' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '7f5bc909c29d4e93d8ccfdda516e51ed44930ee1': '752c52134dcbf2fff13c7be1ce4e9e5dbf428a59', # noqa '38d81702cb28db4f1a6821e64321e5825d1f7fd6': '39c813fb4717a4864bacefbd90b51a3241ae4140', # noqa '99c27ebbd43feca179ac0e895af131d8314cafe1': '3397ca7f709639cbd36b18a0d1b70bce80018c45', # noqa '902f29b4323a9b9de3af6d28e72dd581e76d9397': 'c4e12483f0a13e6851459295a4ae735eb4e4b5c4', # noqa last_revision: 'fd24a76c87a3207428e06612b49860fc78e9f6dc' # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) class SWHSvnLoaderUpdateAndTestCornerCasesAboutEolITTest(BaseTestSvnLoader): def setUp(self): super().setUp(archive_name='pkg-gourmet-with-eol-corner-cases.tgz') self.origin = {'id': 2, 'type': 'svn', 'url': 'file:///dev/null'} self.origin_visit = { 'origin': self.origin['id'], 'visit': 1, } self.loader = SWHSvnLoaderUpdateLessRecentNoStorage() self.loader.prepare( self.svn_mirror_url, self.destination_path, self.origin) @istest def process_repository(self): """EOL corner cases and update. """ previous_unfinished_revision = { 'id': hashutil.hash_to_bytes( '171dc35522bfd17dda4e90a542a0377fb2fc707a'), 'parents': [hashutil.hash_to_bytes( '902f29b4323a9b9de3af6d28e72dd581e76d9397')], 'directory': hashutil.hash_to_bytes( 'fd24a76c87a3207428e06612b49860fc78e9f6dc'), 'target_type': 'revision', 'metadata': { 'extra_headers': [ ['svn_repo_uuid', '3187e211-bb14-4c82-9596-0b59d67cd7f4'], ['svn_revision', '11'] ] } } # when self.loader.process_repository( self.origin_visit, last_known_swh_revision=previous_unfinished_revision) # then # we got the previous run's last revision (rev 11) # so 8 new self.assertEquals(len(self.loader.all_revisions), 8) self.assertEquals(len(self.loader.all_releases), 0) last_revision = '0148ae3eaa520b73a50802c59f3f416b7a36cf8c' # cf. test_loader.org for explaining from where those hashes # come from expected_revisions = { # revision hash | directory hash '027e8769f4786597436ab94a91f85527d04a6cbb': '2d9ca72c6afec6284fb01e459588cbb007017c8c', # noqa '4474d96018877742d9697d5c76666c9693353bfc': 'ab111577e0ab39e4a157c476072af48f2641d93f', # noqa '97ad21eab92961e2a22ca0285f09c6d1e9a7ffbc': 'ab111577e0ab39e4a157c476072af48f2641d93f', # noqa 'd04ea8afcee6205cc8384c091bfc578931c169fd': 'b0a648b02e55a4dce356ac35187a058f89694ec7', # noqa 'ded78810401fd354ffe894aa4a1e5c7d30a645d1': 'b0a648b02e55a4dce356ac35187a058f89694ec7', # noqa '4ee95e39358712f53c4fc720da3fafee9249ed19': 'c3c98df624733fef4e592bef983f93e2ed02b179', # noqa 'ffa901b69ca0f46a2261f42948838d19709cb9f8': 'c3c98df624733fef4e592bef983f93e2ed02b179', # noqa last_revision: '844d4646d6c2b4f3a3b2b22ab0ee38c7df07bab2', # noqa } for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) diff --git a/version.txt b/version.txt index 43b4087..a51063c 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.26-0-g54045a6 \ No newline at end of file +v0.0.27-0-g204c90e \ No newline at end of file