diff --git a/PKG-INFO b/PKG-INFO index 6c92246..e58d9c3 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,61 +1,61 @@ Metadata-Version: 2.1 Name: swh.loader.svn -Version: 0.10.0 +Version: 0.10.1 Summary: Software Heritage Loader SVN Home-page: https://forge.softwareheritage.org/diffusion/DLDSVN Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-loader-svn Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-loader-svn/ Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing License-File: LICENSE License-File: AUTHORS swh-loader-svn ============== The Software Heritage SVN Loader is a tool and a library to walk a remote svn repository and inject into the SWH dataset all contained files that weren't known before. The main entry points are - :class:`swh.loader.svn.loader.SvnLoader` for the main svn loader which ingests content out of a remote svn repository - :class:`swh.loader.svn.loader.SvnLoaderFromDumpArchive` which mounts a repository out of a svn dump prior to ingest it. - :class:`swh.loader.svn.loader.SvnLoaderFromRemoteDump` which mounts a repository with svnrdump prior to ingest its content. # CLI run With the configuration: /tmp/loader_svn.yml: ``` storage: cls: remote args: url: http://localhost:5002/ ``` Run: ``` swh loader --config-file /tmp/loader_svn.yml \ run svn ``` diff --git a/debian/changelog b/debian/changelog index cfaf21b..c8bf9b1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,750 +1,752 @@ -swh-loader-svn (0.10.0-1~swh1~bpo10+1) buster-swh; urgency=medium +swh-loader-svn (0.10.1-1~swh1) unstable-swh; urgency=medium - * Rebuild for buster-swh + * New upstream release 0.10.1 - (tagged by Antoine Lambert + on 2021-11-25 15:20:27 +0100) + * Upstream changes: - version 0.10.1 - -- Software Heritage autobuilder (on jenkins-debian1) Fri, 19 Nov 2021 11:28:36 +0000 + -- Software Heritage autobuilder (on jenkins-debian1) Thu, 25 Nov 2021 14:26:34 +0000 swh-loader-svn (0.10.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.10.0 - (tagged by Antoine Lambert on 2021-11-19 11:58:43 +0100) * Upstream changes: - version 0.10.0 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 19 Nov 2021 11:03:18 +0000 swh-loader-svn (0.9.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.9.1 - (tagged by Antoine R. Dumont (@ardumont) on 2021-11-18 14:19:23 +0100) * Upstream changes: - v0.9.1 - ra: Prevent erroneous removing of file state -- Software Heritage autobuilder (on jenkins-debian1) Thu, 18 Nov 2021 13:22:06 +0000 swh-loader-svn (0.9.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.9.0 - (tagged by Antoine Lambert on 2021-11-17 15:11:56 +0100) * Upstream changes: - version 0.9.0 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 17 Nov 2021 14:14:34 +0000 swh-loader-svn (0.8.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.8.0 - (tagged by Antoine R. Dumont (@ardumont) on 2021-11-09 12:25:09 +0100) * Upstream changes: - v0.8.0 - SvnLoaderFromRemoteDump: Drop dump when svn repository is mounted - loader: Let log statement do the formatting - svn: Modify default loader to use the same as production task -- Software Heritage autobuilder (on jenkins-debian1) Tue, 09 Nov 2021 11:27:49 +0000 swh-loader-svn (0.7.3-1~swh1) unstable-swh; urgency=medium * New upstream release 0.7.3 - (tagged by Antoine R. Dumont (@ardumont) on 2021-11-08 17:29:05 +0100) * Upstream changes: - v0.7.3 - loader: Implement post_load to check for revision divergence - ra: Fix export of non link file with svn:special property set - loader: Fix error when first revision number is not one - loader: Perform a complete reloading when detecting altered history - ra: Handle invalid values for svn:eol-style property - ra: Handle unsetting svn:special property on svn links - ra: Do not process end of lines for svn links - ra: Fix edge case in svn:eol-style property handling - loader: Rename start_from_scratch parameter to incremental - loader: Let logging instructions do the formatting - loader: Fix SvnLoaderFromDumpArchive and add test - Remove reference to deprecated swh.model.identifiers module -- Software Heritage autobuilder (on jenkins-debian1) Mon, 08 Nov 2021 16:32:19 +0000 swh-loader-svn (0.7.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.7.2 - (tagged by Antoine R. Dumont (@ardumont) on 2021-09-30 10:07:44 +0200) * Upstream changes: - v0.7.2 - SvnLoaderFromRemoteDump: Fix failed visit which should be an uneventful visit - svn: Capture svnrdump failure as not_found instead of failed - mypy: Fix errors with release >= v0.900 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 30 Sep 2021 08:10:27 +0000 swh-loader-svn (0.7.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.7.1 - (tagged by Antoine Lambert on 2021-04-29 14:31:33 +0200) * Upstream changes: - version 0.7.1 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 29 Apr 2021 12:34:51 +0000 swh-loader-svn (0.7.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.7.0 - (tagged by Antoine R. Dumont (@ardumont) on 2021-02-17 18:19:35 +0100) * Upstream changes: - v0.7.0 - loader: Expect visit_date as an optional date in constructors - Rework loader instantiation logic according to loader core api -- Software Heritage autobuilder (on jenkins-debian1) Wed, 17 Feb 2021 17:23:16 +0000 swh-loader-svn (0.6.0-1~swh2) unstable-swh; urgency=medium * Update dependencies -- Antoine R. Dumont (@ardumont) Fri, 12 Feb 2021 18:05:59 +0100 swh-loader-svn (0.6.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.6.0 - (tagged by Antoine R. Dumont (@ardumont) on 2021-02-12 17:31:53 +0100) * Upstream changes: - v0.6.0 - svn.loader: Mark visit status as not_found when relevant - svn.loader: Mark visit status as failed - Drop the retrying dependency - tox.ini: pin black to the pre-commit version (19.10b0) to avoid flip-flops -- Software Heritage autobuilder (on jenkins-debian1) Fri, 12 Feb 2021 16:33:45 +0000 swh-loader-svn (0.5.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.5.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-02 14:16:50 +0200) * Upstream changes: - v0.5.0 - svn.loader: Migrate away from SWHConfig mixin - tests: Don't check the number of created 'person' objects - python: Reorder imports with isort - pre- commit: Add isort hook and configuration - pre-commit: Update flake8 hook configuration -- Software Heritage autobuilder (on jenkins-debian1) Fri, 02 Oct 2020 12:17:49 +0000 swh-loader-svn (0.4.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.4.1 - (tagged by Antoine Lambert on 2020-09-08 17:48:55 +0200) * Upstream changes: - version 0.4.1 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 08 Sep 2020 15:53:11 +0000 swh-loader-svn (0.4.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.4.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-09-04 14:05:14 +0200) * Upstream changes: - v0.4.0 - Adapt storage.revision_get calls according to latest api change - Tell pytest not to recurse in dotdirs. -- Software Heritage autobuilder (on jenkins-debian1) Fri, 04 Sep 2020 12:07:00 +0000 swh-loader-svn (0.3.3-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.3 - (tagged by Antoine R. Dumont (@ardumont) on 2020-08-24 16:00:11 +0200) * Upstream changes: - v0.3.3 - svn.loader: Simplify and align incremental visit algorithm -- Software Heritage autobuilder (on jenkins-debian1) Mon, 24 Aug 2020 14:01:22 +0000 swh-loader-svn (0.3.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.2 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-28 08:07:12 +0200) * Upstream changes: - v0.3.2 - Migrate to setuptools-scm - Add missing conftest.py in MANIFEST.in -- Software Heritage autobuilder (on jenkins-debian1) Tue, 28 Jul 2020 06:09:53 +0000 swh-loader-svn (0.3.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.1 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-28 07:50:03 +0200) * Upstream changes: - v0.3.1 - loader: Update swh.storage.origin_get call to latest api change - tests: Reuse pytest fixtures from swh.loader.core - tests: Check against snapshot model object - svn.loader: Drop `build_swh_snapshot` function, oneliner used once - Fix branches types in tests - Clean up the swh.scheduler / swh.storage pytest plugin imports -- Software Heritage autobuilder (on jenkins-debian1) Tue, 28 Jul 2020 05:52:35 +0000 swh-loader-svn (0.3.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.0 - (tagged by David Douard on 2020-07-08 17:18:02 +0200) * Upstream changes: - v0.3.0 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 08 Jul 2020 15:22:12 +0000 swh-loader-svn (0.2.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.1 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-03 15:17:44 +0200) * Upstream changes: - v0.2.1 - svn.loader: Fix uneventful visit case - svn.loader: Fix edge case on mistyped snapshot - tests: Refactor to use pytest - Reuse swh.model.from_disk.iter_directory function - Migrate to Use the object's object_type field when computing objects -- Software Heritage autobuilder (on jenkins-debian1) Fri, 03 Jul 2020 13:20:20 +0000 swh-loader-svn (0.2.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-06-23 15:25:21 +0200) * Upstream changes: - v0.2.0 - loader: Read snapshot out of the last origin visit status - tests: Refactor tests to use assert_last_visit_matches -- Software Heritage autobuilder (on jenkins-debian1) Tue, 23 Jun 2020 13:26:50 +0000 swh-loader-svn (0.1.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.1.2 - (tagged by Antoine R. Dumont (@ardumont) on 2020-06-09 11:03:56 +0200) * Upstream changes: - v0.1.2 - test_loader: assert returned loader status is what we expect - Adapt to swh.model 0.3 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 09 Jun 2020 09:06:01 +0000 swh-loader-svn (0.1.1-1~swh3) unstable-swh; urgency=medium * Enable build for python3.8 (Debian ships a patched subvertpy.) -- Nicolas Dandrimont Wed, 03 Jun 2020 16:27:53 +0200 swh-loader-svn (0.1.1-1~swh2) unstable-swh; urgency=medium * Force removing __init__.py files. -- Nicolas Dandrimont Wed, 03 Jun 2020 16:13:31 +0200 swh-loader-svn (0.1.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.1.1 - (tagged by Antoine Lambert on 2020-06-03 11:49:31 +0200) * Upstream changes: - version 0.1.1 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 03 Jun 2020 09:53:23 +0000 swh-loader-svn (0.1.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.1.0 - (tagged by Nicolas Dandrimont on 2020-06-02 18:54:30 +0200) * Upstream changes: - Release swh.loader.svn v0.1.0 - Blacken all files - Fix compatibility with swh.loader.core v0.1.0 - Fix parameter ordering for loader tasks -- Software Heritage autobuilder (on jenkins-debian1) Tue, 02 Jun 2020 16:58:55 +0000 swh-loader-svn (0.0.54-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.54 - (tagged by Antoine R. Dumont (@ardumont) on 2020-03-27 07:52:30 +0100) * Upstream changes: - v0.0.54 - svn.loader: Let core loader update snapshot & visit at the same time - test_loader: Add missing assertions on visit status - Fix crash on None snapshot -- Software Heritage autobuilder (on jenkins-debian1) Fri, 27 Mar 2020 06:57:15 +0000 swh-loader-svn (0.0.53-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.53 - (tagged by Antoine R. Dumont (@ardumont) on 2020-03-16 13:38:29 +0100) * Upstream changes: - v0.0.53 - Migrate to latest swh-loader- core api change - Move Person parsing to swh-model. -- Software Heritage autobuilder (on jenkins-debian1) Mon, 16 Mar 2020 12:41:14 +0000 swh-loader-svn (0.0.52-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.52 - (tagged by Antoine R. Dumont (@ardumont) on 2020-03-02 15:39:28 +0100) * Upstream changes: - v0.0.52 - loader.svn: Use swh.model objects within the loader - tox.ini: Add debugging dev tool in dedicated environment - converters: Clean up some more dead code - converters: Clean up dead code -- Software Heritage autobuilder (on jenkins-debian1) Mon, 02 Mar 2020 14:42:36 +0000 swh-loader-svn (0.0.51-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.51 - (tagged by Nicolas Dandrimont on 2020-02-19 20:01:31 +0100) * Upstream changes: - Release swh.loader.svn v0.0.51 - Ensure snapshots are properly linked by origin_visits - Update tests for swh.storage v0.0.174 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 19 Feb 2020 19:20:03 +0000 swh-loader-svn (0.0.50-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.50 - (tagged by Antoine R. Dumont (@ardumont) on 2020-02-06 18:45:10 +0100) * Upstream changes: - v0.0.50 - loader.svn: Call storage's (skipped_)content_add endpoints - Fix sphinx warnings -- Software Heritage autobuilder (on jenkins-debian1) Thu, 06 Feb 2020 17:57:08 +0000 swh-loader-svn (0.0.49-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.49 - (tagged by Antoine R. Dumont (@ardumont) on 2020-01-15 15:59:52 +0100) * Upstream changes: - v0.0.49 - svn.loader: Adapt according to latest loader-core simplification -- Software Heritage autobuilder (on jenkins-debian1) Wed, 15 Jan 2020 15:03:50 +0000 swh-loader-svn (0.0.48-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.48 - (tagged by Antoine R. Dumont (@ardumont) on 2019-12-12 14:14:12 +0100) * Upstream changes: - v0.0.48 - svn.loader: Migrate to new buffer/filter behavior -- Software Heritage autobuilder (on jenkins-debian1) Thu, 12 Dec 2019 13:17:31 +0000 swh-loader-svn (0.0.47-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.47 - (tagged by Antoine R. Dumont (@ardumont) on 2019-12-12 11:18:45 +0100) * Upstream changes: - v0.0.47 - tasks: Enforce kwargs use in task message -- Software Heritage autobuilder (on jenkins-debian1) Thu, 12 Dec 2019 10:21:55 +0000 swh-loader-svn (0.0.46-1~swh2) unstable-swh; urgency=medium * Fix build for python3.7 only -- Antoine R. Dumont (@ardumont) Sat, 16 Feb 2019 02:43:37 +0100 swh-loader-svn (0.0.46-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.46 - (tagged by Antoine R. Dumont (@ardumont) on 2019-12-10 10:30:35 +0100) * Upstream changes: - v0.0.46 - tests: Migrate to use pytest- mock's mocker fixture - tasks: Use celery's shared_task decorator - loader.svn: Register svn worker - svn.tasks: Rename tasks according to production's - tasks: Adapt tasks according to the constructor unification - svn: Unify loader constructor with url as parameter - Fix a typo reported by codespell - Add a pre-commit config file - Migrate tox.ini to extras = xxx instead of deps = .[testing] - De-specify testenv:py3 - Include all requirements in MANIFEST.in -- Software Heritage autobuilder (on jenkins-debian1) Tue, 10 Dec 2019 09:38:45 +0000 swh-loader-svn (0.0.45-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.45 - (tagged by Stefano Zacchiroli on 2019-10-10 14:43:18 +0200) * Upstream changes: - v0.0.45 - * typing: minimal changes to make a no-op mypy run pass - * tox.ini: Fix py3 environment to use packaged tests - * Add attribute visit_type. - * Fix warning about pytest.mark.fs. - * add code of conduct document - * CONTRIBUTORS: add Ishan Bhanuka - * Remove unnecessary noqa - * Update dependency on swh-storage. - * Update argument name. - * Use origin urls instead of origin ids. - * update requirements - * Update coverage gitignore - * SvnLoaderFromRemoteDump: Add some debug output when svnrdump is running - * Remove debian packaging from master branch -- Software Heritage autobuilder (on jenkins-debian1) Thu, 10 Oct 2019 12:46:37 +0000 swh-loader-svn (0.0.44-1~swh3) unstable-swh; urgency=low * New upstream release (fixing the distribution) -- Antoine Romain Dumont Sat, 16 Feb 2019 02:43:37 +0100 swh-loader-svn (0.0.44-1~swh2) unstable; urgency=low * New upstream release to fix debian packaging -- Antoine Romain Dumont Sat, 16 Feb 2019 02:19:27 +0100 swh-loader-svn (0.0.44-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.44 - (tagged by David Douard on 2019-02-07 17:35:43 +0100) * Upstream changes: - v0.0.44 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 07 Feb 2019 16:41:45 +0000 swh-loader-svn (0.0.42-1~swh1) unstable-swh; urgency=medium * v0.0.43 * loader: Rename default branch as HEAD * svn.tests: Refactor using loader-core mixins * setup: prepare for pypi upload * Add requirements-test.txt * docs: add title and brief module description -- Antoine R. Dumont (@ardumont) Thu, 11 Oct 2018 14:52:54 +0200 swh-loader-svn (0.0.41-1~swh1) unstable-swh; urgency=medium * v0.0.41 * Fix UnicodeDecodeError when decoding user-defined svn properties * preventing the origin ingestion to finish (T946) * Fix CR/LF issue preventing the origin ingestion to finish (T570) * svn.loader: Add more efficient loader based on remote dumps * svn.loader: Refactoring (align loader with other, name conventions, * improve maintenance, simplify some code, etc...) -- Antoine R. Dumont (@ardumont) Fri, 28 Sep 2018 12:16:31 +0200 swh-loader-svn (0.0.40-1~swh1) unstable-swh; urgency=medium * v0.0.40 * svn/loader: Remove wrong parameter name in dangling clean up call * svn/loader: Refactor: Reuse loader.core.utils.clean_dangling_folders * svn/tasks: Explicit tasks' arguments * svn/loader: Add pre-cleanup step to clean potential dangling folders * d/control: Bump to latest python3-swh.loader.core -- Antoine R. Dumont (@ardumont) Fri, 09 Mar 2018 14:51:09 +0100 swh-loader-svn (0.0.39-1~swh1) unstable-swh; urgency=medium * v0.0.39 * Improve origin_visit initialization step * Properly sandbox the prepare statement so that if it breaks, we can * update appropriately the visit with the correct status -- Antoine R. Dumont (@ardumont) Wed, 07 Mar 2018 11:31:34 +0100 swh-loader-svn (0.0.38-1~swh1) unstable-swh; urgency=medium * v0.0.38 * In failure mode, try harder to make snapshot -- Antoine R. Dumont (@ardumont) Fri, 09 Feb 2018 14:13:30 +0100 swh-loader-svn (0.0.37-1~swh1) unstable-swh; urgency=medium * v0.0.37 * swh.loader.svn: Detect eol inconsistency and make the loading fail * swh.loader.svn: Reduce log verbosity * swh.loader.svn: Migrate exception to their specific module -- Antoine R. Dumont (@ardumont) Thu, 08 Feb 2018 17:57:15 +0100 swh-loader-svn (0.0.36-1~swh1) unstable-swh; urgency=medium * v0.0.36 * swh.loader.svn: Fix corner edge case on symbolic link * swh.loader.svn: Deal with empty svn repository edge case -- Antoine R. Dumont (@ardumont) Wed, 07 Feb 2018 12:36:45 +0100 swh-loader-svn (0.0.35-1~swh1) unstable-swh; urgency=medium * Release swh.loader.svn v0.0.35 * Update to use snapshots instead of occurrences * Properly return loader status to scheduler -- Nicolas Dandrimont Tue, 06 Feb 2018 14:40:28 +0100 swh-loader-svn (0.0.34-1~swh1) unstable-swh; urgency=medium * v0.0.34 * Unify temporary directory naming -- Antoine R. Dumont (@ardumont) Fri, 15 Dec 2017 12:26:50 +0100 swh-loader-svn (0.0.33-1~swh1) unstable-swh; urgency=medium * v0.0.33 * swh.loader.svn: Add option to load a repository from scratch * swh.loader.svn.loader: Fix special symlink case -- Antoine R. Dumont (@ardumont) Fri, 08 Dec 2017 17:38:37 +0100 swh-loader-svn (0.0.32-1~swh1) unstable-swh; urgency=medium * v0.0.32 * d/control: Bump dependency to latest swh-loader-core * tests: Add commits with wrong symbolic links (broken, empty space) * ra: Fix not cleaned up path leading to file exists error * docs: add absolute anchor to documentation index -- Antoine R. Dumont (@ardumont) Wed, 08 Nov 2017 13:22:42 +0100 swh-loader-svn (0.0.31-1~swh1) unstable-swh; urgency=medium * v0.0.31 * swh.loader.svn.ra: Fix blank space in symlink path split problem * swh.loader.svn.ra: Fix missing case about existing directory * swh.loader.svn.ra: Explicit the closure * swh.loader.svn: Deal with origin creation inhibition in base class * swh.loader.svn: Remove unneeded instruction * swh.loader.svn: Fix initializing the origin multiple times * swh.loader.svn: Remove unneeded except clause * swh.loader.svn: uneventful visit is not an error -- Antoine R. Dumont (@ardumont) Tue, 24 Oct 2017 16:35:01 +0200 swh-loader-svn (0.0.30-1~swh1) unstable-swh; urgency=medium * Release swh.loader.svn version 0.0.30 * Update packaging runes -- Nicolas Dandrimont Thu, 12 Oct 2017 18:07:58 +0200 swh-loader-svn (0.0.29-1~swh1) unstable-swh; urgency=medium * v0.0.29 * swh.loader.svn.loader: Fix missing revision id key (which can lead * to an empty partial visit for example in the case of a repository * holding an svn:externals property). -- Antoine R. Dumont (@ardumont) Mon, 09 Oct 2017 16:02:23 +0200 swh-loader-svn (0.0.28-1~swh1) unstable-swh; urgency=medium * Release swh.loader.svn v0.0.28 * Replace swh.model.git with swh.model.from_disk -- Nicolas Dandrimont Fri, 06 Oct 2017 17:18:57 +0200 swh-loader-svn (0.0.27-1~swh1) unstable-swh; urgency=medium * v0.0.27 * d/control: Bump dependency to latest swh.loader.core * swh.loader.svn: Add args to raised exception in edge cases * swh-hashtree: Fix no longer existing swh.core.hashutil import * docs/: add sphinx apidoc generation skeleton -- Antoine R. Dumont (@ardumont) Tue, 03 Oct 2017 19:01:26 +0200 swh-loader-svn (0.0.26-1~swh1) unstable-swh; urgency=medium * Release swh.loader.svn 0.0.26 * update tasks to new swh.scheduler API -- Nicolas Dandrimont Mon, 12 Jun 2017 18:25:40 +0200 swh-loader-svn (0.0.25-1~swh1) unstable-swh; urgency=medium * v0.0.25 * Add blake2s256 new hash computation on content * Use gzip instead of pigz * Migrate from swh.core.hashutil to swh.model.hashutil -- Antoine R. Dumont (@ardumont) Tue, 04 Apr 2017 14:18:57 +0200 swh-loader-svn (0.0.24-1~swh1) unstable-swh; urgency=medium * v0.0.24 * Simplify loader's logic -- Antoine R. Dumont (@ardumont) Wed, 22 Feb 2017 14:09:15 +0100 swh-loader-svn (0.0.23-1~swh1) unstable-swh; urgency=medium * Release swh.loader.svn v0.0.23 * Return timestamps as pairs of integers (Close T680) -- Nicolas Dandrimont Tue, 14 Feb 2017 19:40:27 +0100 swh-loader-svn (0.0.22-1~swh1) unstable-swh; urgency=medium * v0.0.22 * Pass optionally the visit date to producer -- Antoine R. Dumont (@ardumont) Tue, 10 Jan 2017 15:03:45 +0100 swh-loader-svn (0.0.21-1~swh1) unstable-swh; urgency=medium * v0.0.21 * Use the dump's fetch date as visit date -- Antoine R. Dumont (@ardumont) Tue, 10 Jan 2017 15:03:24 +0100 swh-loader-svn (0.0.20-1~swh1) unstable-swh; urgency=medium * v0.0.20 * Fix function call -- Antoine R. Dumont (@ardumont) Fri, 06 Jan 2017 15:38:14 +0100 swh-loader-svn (0.0.19-1~swh1) unstable-swh; urgency=medium * v0.0.19 * Rename task_queue adequately * Fix - Remove empty temporary directory in corner cases * Fix generator issue when using local storage -- Antoine R. Dumont (@ardumont) Fri, 06 Jan 2017 14:10:19 +0100 swh-loader-svn (0.0.18-1~swh1) unstable-swh; urgency=medium * v0.0.18 * Add check revision hash computation divergence during loading * Add task to load a svn repository from an archive dump and load the * repository * Raise error when svn:externals keyword is encountered (T610) since * it's not dealt with yet. * Revert modifications related to svn:eol-style since no good policy has * been found yet to deal with it -- Antoine R. Dumont (@ardumont) Thu, 15 Dec 2016 10:57:26 +0100 swh-loader-svn (0.0.17-1~swh1) unstable-swh; urgency=medium * v0.0.17 * Force eol conversion when needed -- Antoine R. Dumont (@ardumont) Thu, 08 Dec 2016 18:25:55 +0100 swh-loader-svn (0.0.16-1~swh1) unstable-swh; urgency=medium * v0.0.16 * Improve origin_visit policy -- Antoine R. Dumont (@ardumont) Fri, 26 Aug 2016 15:59:31 +0200 swh-loader-svn (0.0.15-1~swh1) unstable-swh; urgency=medium * v0.0.15 * Update svn loader to register origin_visit's state -- Antoine R. Dumont (@ardumont) Wed, 24 Aug 2016 14:56:54 +0200 swh-loader-svn (0.0.14-1~swh1) unstable-swh; urgency=medium * v0.0.14 * Update svn loader to register origin_visit's state -- Antoine R. Dumont (@ardumont) Tue, 23 Aug 2016 16:38:16 +0200 swh-loader-svn (0.0.13-1~swh1) unstable-swh; urgency=medium * v0.0.13 -- Antoine R. Dumont (@ardumont) Thu, 18 Aug 2016 10:30:13 +0200 swh-loader-svn (0.0.12-1~swh1) unstable-swh; urgency=medium * v0.0.12 * Inhibit keyword expansion during export phase (svn update) -- Antoine R. Dumont (@ardumont) Wed, 06 Jul 2016 16:17:51 +0200 swh-loader-svn (0.0.11-1~swh1) unstable-swh; urgency=medium * v0.0.11 * Reschedule when error on loading an svn repository * Start from last known revision * Permit to load local mirror with remote mirror origin -- Antoine R. Dumont (@ardumont) Fri, 01 Jul 2016 16:51:37 +0200 swh-loader-svn (0.0.10-1~swh1) unstable-swh; urgency=medium * v0.0.10 * Add retry policy around britle action (connection to remote svn server) -- Antoine R. Dumont (@ardumont) Sat, 25 Jun 2016 11:30:34 +0200 swh-loader-svn (0.0.9-1~swh1) unstable-swh; urgency=medium * v0.0.9 * Restrict loader to 2 possible policies: swh, git-svn * Improve coverage * Update README -- Antoine R. Dumont (@ardumont) Fri, 24 Jun 2016 13:56:54 +0200 swh-loader-svn (0.0.8-1~swh1) unstable-swh; urgency=medium * v0.0.8 * Use remote-access approach to hash and compute hashes from svn * repository (migrate major dependency from pysvn to subvertpy) -- Antoine R. Dumont (@ardumont) Sat, 11 Jun 2016 19:55:54 +0200 swh-loader-svn (0.0.7-1~swh1) unstable-swh; urgency=medium * v0.0.7 * d/control: Bump dependency version to latest swh-model -- Antoine R. Dumont (@ardumont) Thu, 26 May 2016 16:10:32 +0200 swh-loader-svn (0.0.6-1~swh1) unstable-swh; urgency=medium * v0.0.6 * d/control: Bump swh-model dependency * Simplify ignore folder corner cases * Rework log -- Antoine R. Dumont (@ardumont) Wed, 25 May 2016 23:53:10 +0200 swh-loader-svn (0.0.5-1~swh1) unstable-swh; urgency=medium * v0.0.5 * Add svn update behavior as option (default to True) * d/control: Bump dependency version to swh-loader-core -- Antoine R. Dumont (@ardumont) Wed, 25 May 2016 12:44:35 +0200 swh-loader-svn (0.0.4-1~swh1) unstable-swh; urgency=medium * v0.0.4 * Add options for git-svn like behavior (git svn clone --no- metadata) * Ignore empty folder * Add extra-line in svn commit message * No revision metadata (beware that this break the update repo * mechanism) * Also, use @ pattern for committer/author -- Antoine R. Dumont (@ardumont) Tue, 24 May 2016 17:04:40 +0200 swh-loader-svn (0.0.3-1~swh1) unstable-swh; urgency=medium * v0.0.3 * Improve filesystem cleanup (before: left empty directory) * Use pysvn to extract uuid information (before: subprocess with subversion call) * Keep the working copy's name on the local checkout -- Antoine R. Dumont (@ardumont) Fri, 15 Apr 2016 16:10:51 +0200 swh-loader-svn (0.0.2-1~swh1) unstable-swh; urgency=medium * v0.0.2 * Update to the right swh dependencies. -- Antoine R. Dumont (@ardumont) Fri, 15 Apr 2016 13:02:04 +0200 swh-loader-svn (0.0.1-1~swh1) unstable-swh; urgency=medium * Initial release * v0.0.1 -- Antoine R. Dumont (@ardumont) Wed, 13 Apr 2016 16:46:37 +0200 diff --git a/swh.loader.svn.egg-info/PKG-INFO b/swh.loader.svn.egg-info/PKG-INFO index 6c92246..e58d9c3 100644 --- a/swh.loader.svn.egg-info/PKG-INFO +++ b/swh.loader.svn.egg-info/PKG-INFO @@ -1,61 +1,61 @@ Metadata-Version: 2.1 Name: swh.loader.svn -Version: 0.10.0 +Version: 0.10.1 Summary: Software Heritage Loader SVN Home-page: https://forge.softwareheritage.org/diffusion/DLDSVN Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-loader-svn Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-loader-svn/ Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing License-File: LICENSE License-File: AUTHORS swh-loader-svn ============== The Software Heritage SVN Loader is a tool and a library to walk a remote svn repository and inject into the SWH dataset all contained files that weren't known before. The main entry points are - :class:`swh.loader.svn.loader.SvnLoader` for the main svn loader which ingests content out of a remote svn repository - :class:`swh.loader.svn.loader.SvnLoaderFromDumpArchive` which mounts a repository out of a svn dump prior to ingest it. - :class:`swh.loader.svn.loader.SvnLoaderFromRemoteDump` which mounts a repository with svnrdump prior to ingest its content. # CLI run With the configuration: /tmp/loader_svn.yml: ``` storage: cls: remote args: url: http://localhost:5002/ ``` Run: ``` swh loader --config-file /tmp/loader_svn.yml \ run svn ``` diff --git a/swh/loader/svn/svn.py b/swh/loader/svn/svn.py index d8ba038..3e1ee5d 100644 --- a/swh/loader/svn/svn.py +++ b/swh/loader/svn/svn.py @@ -1,309 +1,296 @@ # Copyright (C) 2015-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """SVN client in charge of iterating over svn logs and yield commit representations including the hash tree/content computations per svn commit. """ import logging import os import shutil import tempfile from typing import Dict, Iterator, List, Optional, Tuple, Union from subvertpy import client, properties from subvertpy.ra import Auth, RemoteAccess, get_username_provider from swh.model.from_disk import Directory as DirectoryFromDisk from swh.model.model import ( Content, Directory, Person, SkippedContent, TimestampWithTimezone, ) from . import converters, ra # When log message contains empty data DEFAULT_AUTHOR_MESSAGE = "" logger = logging.getLogger(__name__) class SvnRepo: """Svn repository representation. Args: remote_url: Remove svn repository url origin_url: Associated origin identifier local_dirname: Path to write intermediary svn action results """ def __init__( self, remote_url: str, origin_url: str, local_dirname: str, max_content_length: int, ): self.remote_url = remote_url.rstrip("/") self.origin_url = origin_url auth = Auth([get_username_provider()]) # one connection for log iteration self.conn_log = RemoteAccess(self.remote_url, auth=auth) # another for replay self.conn = RemoteAccess(self.remote_url, auth=auth) # one client for update operation self.client = client.Client(auth=auth) self.local_dirname = local_dirname local_name = os.path.basename(self.remote_url) self.local_url = os.path.join(self.local_dirname, local_name).encode("utf-8") self.uuid = self.conn.get_uuid().encode("utf-8") self.swhreplay = ra.Replay(conn=self.conn, rootpath=self.local_url) self.max_content_length = max_content_length def __str__(self): return str( { "swh-origin": self.origin_url, "remote_url": self.remote_url, "local_url": self.local_url, "uuid": self.uuid, } ) def head_revision(self) -> int: """Retrieve current head revision. """ return self.conn.get_latest_revnum() def initial_revision(self) -> int: """Retrieve the initial revision from which the remote url appeared. """ return 1 def convert_commit_message(self, msg: Union[str, bytes]) -> bytes: """Simply encode the commit message. Args: msg: the commit message to convert. Returns: The transformed message as bytes. """ if isinstance(msg, bytes): return msg return msg.encode("utf-8") def convert_commit_date(self, date: str) -> TimestampWithTimezone: """Convert the message commit date into a timestamp in swh format. The precision is kept. Args: date: the commit date to convert. Returns: The transformed date. """ return converters.svn_date_to_swh_date(date) def convert_commit_author(self, author: Optional[bytes]) -> Person: """Convert the commit author into an swh person. Args: author: the commit author to convert. Returns: Person as model object """ return converters.svn_author_to_swh_person(author) def __to_entry(self, log_entry: Tuple) -> Dict: changed_paths, rev, revprops, has_children = log_entry author_date = self.convert_commit_date( revprops.get(properties.PROP_REVISION_DATE) ) author = self.convert_commit_author( revprops.get(properties.PROP_REVISION_AUTHOR) ) message = self.convert_commit_message( revprops.get(properties.PROP_REVISION_LOG, DEFAULT_AUTHOR_MESSAGE) ) return { "rev": rev, "author_date": author_date, "author_name": author, "message": message, } def logs(self, revision_start: int, revision_end: int) -> Iterator[Dict]: """Stream svn logs between revision_start and revision_end by chunks of block_size logs. Yields revision and associated revision information between the revision start and revision_end. Args: revision_start: the svn revision starting bound revision_end: the svn revision ending bound Yields: tuple: tuple of revisions and logs: - revisions: list of revisions in order - logs: Dictionary with key revision number and value the log entry. The log entry is a dictionary with the following keys: - author_date: date of the commit - author_name: name of the author - message: commit message """ for log_entry in self.conn_log.iter_log( paths=None, start=revision_start, end=revision_end, discover_changed_paths=False, ): yield self.__to_entry(log_entry) - def export(self, revision: int) -> None: - """Export the repository to a given version. - - """ - self.client.export( - self.remote_url, - to=self.local_url.decode("utf-8"), - rev=revision, - ignore_keywords=True, - ) - def export_temporary(self, revision: int) -> Tuple[str, bytes]: """Export the repository to a given revision in a temporary location. This is up to the caller of this function to clean up the temporary location when done (cf. self.clean_fs method) Args: revision: Revision to export at Returns: The tuple local_dirname the temporary location root folder, local_url where the repository was exported. """ local_dirname = tempfile.mkdtemp( dir=self.local_dirname, prefix=f"check-revision-{revision}." ) local_name = os.path.basename(self.remote_url) local_url = os.path.join(local_dirname, local_name) self.client.export( self.remote_url, to=local_url, rev=revision, ignore_keywords=True ) return local_dirname, os.fsencode(local_url) def swh_hash_data_per_revision( self, start_revision: int, end_revision: int ) -> Iterator[ Tuple[ int, Optional[int], Dict, Tuple[List[Content], List[SkippedContent], List[Directory]], DirectoryFromDisk, ], ]: """Compute swh hash data per each revision between start_revision and end_revision. Args: start_revision: starting revision end_revision: ending revision Yields: Tuple (rev, nextrev, commit, objects_per_path): - rev: current revision - nextrev: next revision or None if we reached end_revision. - commit: commit data (author, date, message) for such revision - objects_per_path: Tuple of list of objects between start_revision and end_revision - complete Directory representation """ # even in incremental loading mode, we need to replay the whole set of # path modifications from first revision to restore possible file states induced # by setting svn properties on those files (end of line style for instance) first_revision = 1 if start_revision else 0 # handle empty repository edge case for commit in self.logs(first_revision, end_revision): rev = commit["rev"] objects = self.swhreplay.compute_objects(rev) if rev == end_revision: nextrev = None else: nextrev = rev + 1 if rev >= start_revision: # start yielding new data to archive once we reached the revision to # resume the loading from yield rev, nextrev, commit, objects, self.swhreplay.directory def swh_hash_data_at_revision( self, revision: int ) -> Iterator[Tuple[Dict, DirectoryFromDisk]]: """Compute the information at a given svn revision. This is expected to be used - for update only. + for checks only. Yields: The tuple (commit dictionary, targeted directory object). """ # Update disk representation of the repository at revision id - self.export(revision) + local_dirname, local_url = self.export_temporary(revision) # Compute the current hashes on disk directory = DirectoryFromDisk.from_disk( - path=os.fsencode(self.local_url), max_content_length=self.max_content_length - ) - - # Update the replay collaborator with the right state - self.swhreplay = ra.Replay( - conn=self.conn, rootpath=self.local_url, directory=directory + path=local_url, max_content_length=self.max_content_length ) # Retrieve the commit information for revision commit = list(self.logs(revision, revision))[0] + # Clean export directory + self.clean_fs(local_dirname) + yield commit, directory def clean_fs(self, local_dirname: Optional[str] = None) -> None: """Clean up the local working copy. Args: local_dirname: Path to remove recursively if provided. Otherwise, remove the temporary upper root tree used for svn repository loading. """ dirname = local_dirname or self.local_dirname if os.path.exists(dirname): logger.debug("cleanup %s", dirname) shutil.rmtree(dirname) diff --git a/swh/loader/svn/tasks.py b/swh/loader/svn/tasks.py index 0009255..de72c68 100644 --- a/swh/loader/svn/tasks.py +++ b/swh/loader/svn/tasks.py @@ -1,108 +1,111 @@ # Copyright (C) 2015-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime from typing import Optional from celery import shared_task import iso8601 from .loader import SvnLoader, SvnLoaderFromDumpArchive, SvnLoaderFromRemoteDump def convert_to_datetime(date: Optional[str]) -> Optional[datetime]: + if date is None: + return None try: + assert isinstance(date, str) return iso8601.parse_date(date) except Exception: return None @shared_task(name=__name__ + ".LoadSvnRepository") def load_svn( *, url: Optional[str] = None, origin_url: Optional[str] = None, visit_date: Optional[str] = None, incremental: Optional[bool] = True, ): """Import a svn repository Args: url: (mandatory) svn's repository url to ingest data from origin_url: Optional original url override to use as origin reference in the archive. If not provided, "url" is used as origin. visit_date: Optional date to override the visit date incremental: If True, the default, starts from the last snapshot (if any). Otherwise, starts from the initial commit of the repository. """ loader = SvnLoader.from_configfile( url=url, origin_url=origin_url, visit_date=convert_to_datetime(visit_date), incremental=incremental, ) return loader.load() @shared_task(name=__name__ + ".MountAndLoadSvnRepository") def load_svn_from_archive( *, url: Optional[str] = None, archive_path: Optional[str] = None, visit_date: Optional[str] = None, incremental: Optional[bool] = True, ): """1. Mount an svn dump from archive as a local svn repository 2. Load it through the svn loader 3. Clean up mounted svn repository archive Args: url: origin url archive_path: Path on disk to the archive holdin the svn repository to ingest visit_date: Optional date to override the visit date incremental: If True, the default, starts from the last snapshot (if any). Otherwise, starts from the initial commit of the repository. """ loader = SvnLoaderFromDumpArchive.from_configfile( url=url, archive_path=archive_path, visit_date=convert_to_datetime(visit_date), incremental=incremental, ) return loader.load() @shared_task(name=__name__ + ".DumpMountAndLoadSvnRepository") def load_svn_from_remote_dump( *, url: Optional[str] = None, origin_url: Optional[str] = None, visit_date: Optional[str] = None, incremental: Optional[bool] = True, ): """1. Mount a remote svn dump as a local svn repository. 2. Load it through the svn loader. 3. Clean up mounted svn repository archive. Args: url: (mandatory) svn's repository url to ingest data from origin_url: Optional original url override to use as origin reference in the archive. If not provided, "url" is used as origin. visit_date: Optional date to override the visit date incremental: If True, the default, starts from the last snapshot (if any). Otherwise, starts from the initial commit of the repository. """ loader = SvnLoaderFromRemoteDump.from_configfile( url=url, origin_url=origin_url, visit_date=convert_to_datetime(visit_date), incremental=incremental, ) return loader.load() diff --git a/swh/loader/svn/tests/test_loader.py b/swh/loader/svn/tests/test_loader.py index 88ea86f..640266a 100644 --- a/swh/loader/svn/tests/test_loader.py +++ b/swh/loader/svn/tests/test_loader.py @@ -1,1560 +1,1621 @@ # Copyright (C) 2016-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from enum import Enum from io import BytesIO import os import shutil import subprocess from typing import Any, Dict, List import pytest from subvertpy import SubversionException, delta, repos from subvertpy.ra import Auth, RemoteAccess, get_username_provider from typing_extensions import TypedDict from swh.loader.svn.loader import ( SvnLoader, SvnLoaderFromDumpArchive, SvnLoaderFromRemoteDump, ) from swh.loader.svn.svn import SvnRepo from swh.loader.svn.utils import init_svn_repo_from_dump from swh.loader.tests import ( assert_last_visit_matches, check_snapshot, get_stats, prepare_repository_from_archive, ) from swh.model.from_disk import DentryPerms from swh.model.hashutil import hash_to_bytes from swh.model.model import Snapshot, SnapshotBranch, TargetType GOURMET_SNAPSHOT = Snapshot( id=hash_to_bytes("889cacc2731e3312abfb2b1a0c18ade82a949e07"), branches={ b"HEAD": SnapshotBranch( target=hash_to_bytes("4876cb10aec6f708f7466dddf547567b65f6c39c"), target_type=TargetType.REVISION, ) }, ) GOURMET_UPDATES_SNAPSHOT = Snapshot( id=hash_to_bytes("11086d15317014e43d2438b7ffc712c44f1b8afe"), branches={ b"HEAD": SnapshotBranch( target=hash_to_bytes("171dc35522bfd17dda4e90a542a0377fb2fc707a"), target_type=TargetType.REVISION, ) }, ) def test_loader_svn_not_found_no_mock(swh_storage, tmp_path): """Given an unknown repository, the loader visit ends up in status not_found""" repo_url = "unknown-repository" loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) assert loader.load() == {"status": "uneventful"} assert_last_visit_matches( swh_storage, repo_url, status="not_found", type="svn", ) @pytest.mark.parametrize( "exception_msg", ["Unable to connect to a repository at URL", "Unknown URL type",] ) def test_loader_svn_not_found(swh_storage, tmp_path, exception_msg, mocker): """Given unknown repository issues, the loader visit ends up in status not_found""" mock = mocker.patch("swh.loader.svn.loader.SvnRepo") mock.side_effect = SubversionException(exception_msg, 0) unknown_repo_url = "unknown-repository" loader = SvnLoader(swh_storage, unknown_repo_url, temp_directory=tmp_path) assert loader.load() == {"status": "uneventful"} assert_last_visit_matches( swh_storage, unknown_repo_url, status="not_found", type="svn", ) @pytest.mark.parametrize( "exception", [ SubversionException("Irrelevant message, considered a failure", 10), SubversionException("Present but fails to read, considered a failure", 20), ValueError("considered a failure"), ], ) def test_loader_svn_failures(swh_storage, tmp_path, exception, mocker): """Given any errors raised, the loader visit ends up in status failed""" mock = mocker.patch("swh.loader.svn.loader.SvnRepo") mock.side_effect = exception existing_repo_url = "existing-repo-url" loader = SvnLoader(swh_storage, existing_repo_url, temp_directory=tmp_path) assert loader.load() == {"status": "failed"} assert_last_visit_matches( swh_storage, existing_repo_url, status="failed", type="svn", ) def test_loader_svnrdump_not_found(swh_storage, tmp_path, mocker): """Loading from remote dump which does not exist should end up as not_found visit""" unknown_repo_url = "file:///tmp/svn.code.sf.net/p/white-rats-studios/svn" loader = SvnLoaderFromRemoteDump( swh_storage, unknown_repo_url, temp_directory=tmp_path ) assert loader.load() == {"status": "uneventful"} assert_last_visit_matches( swh_storage, unknown_repo_url, status="not_found", type="svn", ) def test_loader_svnrdump_no_such_revision(swh_storage, tmp_path, datadir): """Visit multiple times an origin with the remote loader should not raise. It used to fail the ingestion on the second visit with a "No such revision x, 160006" message. """ archive_ori_dump = os.path.join(datadir, "penguinsdbtools2018.dump.gz") archive_dump_dir = os.path.join(tmp_path, "dump") os.mkdir(archive_dump_dir) archive_dump = os.path.join(archive_dump_dir, "penguinsdbtools2018.dump.gz") # loader now drops the dump as soon as it's mounted so we need to make a copy first shutil.copyfile(archive_ori_dump, archive_dump) loading_path = str(tmp_path / "loading") os.mkdir(loading_path) # Prepare the dump as a local svn repository for test purposes temp_dir, repo_path = init_svn_repo_from_dump( archive_dump, root_dir=tmp_path, gzip=True ) repo_url = f"file://{repo_path}" loader = SvnLoaderFromRemoteDump(swh_storage, repo_url, temp_directory=loading_path) assert loader.load() == {"status": "eventful"} actual_visit = assert_last_visit_matches( swh_storage, repo_url, status="full", type="svn", ) loader2 = SvnLoaderFromRemoteDump( swh_storage, repo_url, temp_directory=loading_path ) # Visiting a second time the same repository should be uneventful... assert loader2.load() == {"status": "uneventful"} actual_visit2 = assert_last_visit_matches( swh_storage, repo_url, status="full", type="svn", ) assert actual_visit.snapshot is not None # ... with the same snapshot as the first visit assert actual_visit2.snapshot == actual_visit.snapshot def test_loader_svn_new_visit(swh_storage, datadir, tmp_path): """Eventful visit should yield 1 snapshot""" archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}.tgz") repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) assert loader.load() == {"status": "eventful"} assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, ) stats = get_stats(loader.storage) assert stats == { "content": 19, "directory": 17, "origin": 1, "origin_visit": 1, "release": 0, "revision": 6, "skipped_content": 0, "snapshot": 1, } check_snapshot(GOURMET_SNAPSHOT, loader.storage) def test_loader_svn_2_visits_no_change(swh_storage, datadir, tmp_path): """Visit multiple times a repository with no change should yield the same snapshot """ archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}.tgz") repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) assert loader.load() == {"status": "eventful"} visit_status1 = assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, ) assert loader.load() == {"status": "uneventful"} visit_status2 = assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, ) assert visit_status1.date < visit_status2.date assert visit_status1.snapshot == visit_status2.snapshot stats = get_stats(loader.storage) assert stats["origin_visit"] == 1 + 1 # computed twice the same snapshot assert stats["snapshot"] == 1 # even starting from previous revision... start_revision = loader.storage.revision_get( [hash_to_bytes("95edacc8848369d6fb1608e887d6d2474fd5224f")] )[0] assert start_revision is not None loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) assert loader.load() == {"status": "uneventful"} stats = get_stats(loader.storage) assert stats["origin_visit"] == 2 + 1 # ... with no change in repository, this yields the same snapshot assert stats["snapshot"] == 1 assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, ) def test_loader_tampered_repository(swh_storage, datadir, tmp_path): """In this scenario, the dump has been tampered with to modify the commit log [1]. This results in a hash divergence which is detected at startup after a new run for the same origin. In effect, this will perform a complete reloading of the repository. [1] Tampering with revision 6 log message following: ``` tar xvf pkg-gourmet.tgz # initial repository ingested cd pkg-gourmet/ echo "Tampering with commit log message for fun and profit" > log.txt svnadmin setlog . -r 6 log.txt --bypass-hooks tar cvf pkg-gourmet-tampered-rev6-log.tgz pkg-gourmet/ ``` """ archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}.tgz") repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) assert loader.load() == {"status": "eventful"} check_snapshot(GOURMET_SNAPSHOT, loader.storage) assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, ) archive_path2 = os.path.join(datadir, "pkg-gourmet-tampered-rev6-log.tgz") repo_tampered_url = prepare_repository_from_archive( archive_path2, archive_name, tmp_path ) loader2 = SvnLoader( swh_storage, repo_tampered_url, origin_url=repo_url, temp_directory=tmp_path ) assert loader2.load() == {"status": "eventful"} assert_last_visit_matches( loader2.storage, repo_url, status="full", type="svn", snapshot=hash_to_bytes("5aa61959e788e281fd6e187053d0f46c68e8d8bb"), ) stats = get_stats(loader.storage) assert stats["origin"] == 1 assert stats["origin_visit"] == 2 assert stats["snapshot"] == 2 def test_loader_svn_visit_with_changes(swh_storage, datadir, tmp_path): """In this scenario, the repository has been updated with new changes. The loading visit should result in new objects stored and 1 new snapshot. """ archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}.tgz") repo_initial_url = prepare_repository_from_archive( archive_path, archive_name, tmp_path ) # repo_initial_url becomes the origin_url we want to visit some more below loader = SvnLoader(swh_storage, repo_initial_url, temp_directory=tmp_path) assert loader.load() == {"status": "eventful"} visit_status1 = assert_last_visit_matches( loader.storage, repo_initial_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, ) archive_path = os.path.join(datadir, "pkg-gourmet-with-updates.tgz") repo_updated_url = prepare_repository_from_archive( archive_path, "pkg-gourmet", tmp_path ) loader = SvnLoader( swh_storage, repo_updated_url, origin_url=repo_initial_url, temp_directory=tmp_path, ) assert loader.load() == {"status": "eventful"} visit_status2 = assert_last_visit_matches( loader.storage, repo_updated_url, status="full", type="svn", snapshot=GOURMET_UPDATES_SNAPSHOT.id, ) assert visit_status1.date < visit_status2.date assert visit_status1.snapshot != visit_status2.snapshot stats = get_stats(loader.storage) assert stats == { "content": 22, "directory": 28, "origin": 1, "origin_visit": 2, "release": 0, "revision": 11, "skipped_content": 0, "snapshot": 2, } check_snapshot(GOURMET_UPDATES_SNAPSHOT, loader.storage) # Let's start the ingestion from the start, this should yield the same result loader = SvnLoader( swh_storage, repo_updated_url, origin_url=repo_initial_url, incremental=False, temp_directory=tmp_path, ) assert loader.load() == {"status": "eventful"} visit_status3 = assert_last_visit_matches( loader.storage, repo_updated_url, status="full", type="svn", snapshot=GOURMET_UPDATES_SNAPSHOT.id, ) assert visit_status2.date < visit_status3.date assert visit_status3.snapshot == visit_status2.snapshot check_snapshot(GOURMET_UPDATES_SNAPSHOT, loader.storage) stats = get_stats(loader.storage) assert stats["origin"] == 1 # always the same visit assert stats["origin_visit"] == 2 + 1 # 1 more visit assert stats["snapshot"] == 2 # no new snapshot def test_loader_svn_visit_start_from_revision(swh_storage, datadir, tmp_path): """Starting from existing revision, next visit on changed repo should yield 1 new snapshot. """ archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}.tgz") repo_initial_url = prepare_repository_from_archive( archive_path, archive_name, tmp_path ) # repo_initial_url becomes the origin_url we want to visit some more below loader = SvnLoader(swh_storage, repo_initial_url, temp_directory=tmp_path) assert loader.load() == {"status": "eventful"} visit_status1 = assert_last_visit_matches( loader.storage, repo_initial_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, ) start_revision = loader.storage.revision_get( [hash_to_bytes("95edacc8848369d6fb1608e887d6d2474fd5224f")] )[0] assert start_revision is not None archive_path = os.path.join(datadir, "pkg-gourmet-with-updates.tgz") repo_updated_url = prepare_repository_from_archive( archive_path, "pkg-gourmet", tmp_path ) # we'll start from start_revision loader = SvnLoader( swh_storage, repo_updated_url, origin_url=repo_initial_url, temp_directory=tmp_path, ) assert loader.load() == {"status": "eventful"} # nonetheless, we obtain the same snapshot (as previous tests on that repository) visit_status2 = assert_last_visit_matches( loader.storage, repo_updated_url, status="full", type="svn", snapshot=GOURMET_UPDATES_SNAPSHOT.id, ) assert visit_status1.date < visit_status2.date assert visit_status1.snapshot != visit_status2.snapshot stats = get_stats(loader.storage) assert stats == { "content": 22, "directory": 28, "origin": 1, "origin_visit": 2, "release": 0, "revision": 11, "skipped_content": 0, "snapshot": 2, } check_snapshot(GOURMET_UPDATES_SNAPSHOT, loader.storage) def test_loader_svn_visit_with_eol_style(swh_storage, datadir, tmp_path): """Check that a svn repo containing a versioned file with CRLF line endings with svn:eol-style property set to 'native' (this is a violation of svn specification as the file should have been stored with LF line endings) can be loaded anyway. """ archive_name = "mediawiki-repo-r407-eol-native-crlf" archive_path = os.path.join(datadir, f"{archive_name}.tgz") repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) assert loader.load() == {"status": "eventful"} mediawiki_snapshot = Snapshot( id=hash_to_bytes("d6d6e9703f157c5702d9a4a5dec878926ed4ab76"), branches={ b"HEAD": SnapshotBranch( target=hash_to_bytes("7da4975c363101b819756d33459f30a866d01b1b"), target_type=TargetType.REVISION, ) }, ) check_snapshot(mediawiki_snapshot, loader.storage) assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", snapshot=mediawiki_snapshot.id, ) stats = get_stats(loader.storage) assert stats["origin"] == 1 assert stats["origin_visit"] == 1 assert stats["snapshot"] == 1 def test_loader_svn_visit_with_mixed_crlf_lf(swh_storage, datadir, tmp_path): """Check that a svn repo containing a versioned file with mixed CRLF/LF line endings with svn:eol-style property set to 'native' (this is a violation of svn specification as mixed line endings for textual content should not be stored when the svn:eol-style property is set) can be loaded anyway. """ archive_name = "pyang-repo-r343-eol-native-mixed-lf-crlf" archive_path = os.path.join(datadir, f"{archive_name}.tgz") repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) assert loader.load() == {"status": "eventful"} pyang_snapshot = Snapshot( id=hash_to_bytes("6d9590de11b00a5801de0ff3297c5b44bbbf7d24"), branches={ b"HEAD": SnapshotBranch( target=hash_to_bytes("9c6962eeb9164a636c374be700672355e34a98a7"), target_type=TargetType.REVISION, ) }, ) check_snapshot(pyang_snapshot, loader.storage) assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", snapshot=pyang_snapshot.id, ) stats = get_stats(loader.storage) assert stats["origin"] == 1 assert stats["origin_visit"] == 1 assert stats["snapshot"] == 1 def test_loader_svn_with_external_properties(swh_storage, datadir, tmp_path): """Repository with svn:external properties cannot be fully ingested yet """ archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, "pkg-gourmet-with-external-id.tgz") repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) assert loader.load() == {"status": "eventful"} gourmet_externals_snapshot = Snapshot( id=hash_to_bytes("19cb68d0a3f22372e2b7017ea5e2a2ea5ae3e09a"), branches={ b"HEAD": SnapshotBranch( target=hash_to_bytes("82a7a4a09f9549223429143ba36ad77375e33c5c"), target_type=TargetType.REVISION, ) }, ) check_snapshot(gourmet_externals_snapshot, loader.storage) assert_last_visit_matches( loader.storage, repo_url, status="partial", type="svn", snapshot=gourmet_externals_snapshot.id, ) stats = get_stats(loader.storage) assert stats["origin"] == 1 assert stats["origin_visit"] == 1 assert stats["snapshot"] == 1 # repository holds 21 revisions, but the last commit holds an 'svn:externals' # property which will make the loader-svn stops at the last revision prior to the # bad one assert stats["revision"] == 21 - 1 # commit with the svn:external property def test_loader_svn_with_symlink(swh_storage, datadir, tmp_path): """Repository with symlinks should be ingested ok Edge case: - first create a file and commit it. Remove it, then add folder holding the same name, commit. - do the same scenario with symbolic link (instead of file) """ archive_name = "pkg-gourmet" archive_path = os.path.join( datadir, "pkg-gourmet-with-edge-case-links-and-files.tgz" ) repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) assert loader.load() == {"status": "eventful"} gourmet_edge_cases_snapshot = Snapshot( id=hash_to_bytes("18e60982fe521a2546ab8c3c73a535d80462d9d0"), branches={ b"HEAD": SnapshotBranch( target=hash_to_bytes("3f43af2578fccf18b0d4198e48563da7929dc608"), target_type=TargetType.REVISION, ) }, ) check_snapshot(gourmet_edge_cases_snapshot, loader.storage) assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", snapshot=gourmet_edge_cases_snapshot.id, ) stats = get_stats(loader.storage) assert stats["origin"] == 1 assert stats["origin_visit"] == 1 assert stats["snapshot"] == 1 assert stats["revision"] == 19 def test_loader_svn_with_wrong_symlinks(swh_storage, datadir, tmp_path): """Repository with wrong symlinks should be ingested ok nonetheless Edge case: - wrong symbolic link - wrong symbolic link with empty space names """ archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, "pkg-gourmet-with-wrong-link-cases.tgz") repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) assert loader.load() == {"status": "eventful"} gourmet_wrong_links_snapshot = Snapshot( id=hash_to_bytes("b17f38acabb90f066dedd30c29f01a02af88a5c4"), branches={ b"HEAD": SnapshotBranch( target=hash_to_bytes("cf30d3bb9d5967d0a2bbeacc405f10a5dd9b138a"), target_type=TargetType.REVISION, ) }, ) check_snapshot(gourmet_wrong_links_snapshot, loader.storage) assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", snapshot=gourmet_wrong_links_snapshot.id, ) stats = get_stats(loader.storage) assert stats["origin"] == 1 assert stats["origin_visit"] == 1 assert stats["snapshot"] == 1 assert stats["revision"] == 21 def test_loader_svn_cleanup_loader(swh_storage, datadir, tmp_path): """Loader should clean up its working directory after the load """ archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}.tgz") repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) loading_temp_directory = str(tmp_path / "loading") os.mkdir(loading_temp_directory) loader = SvnLoader(swh_storage, repo_url, temp_directory=loading_temp_directory) assert loader.load() == {"status": "eventful"} # the root temporary directory still exists assert os.path.exists(loader.temp_directory) # but it should be empty assert os.listdir(loader.temp_directory) == [] def test_loader_svn_cleanup_loader_from_remote_dump(swh_storage, datadir, tmp_path): """Loader should clean up its working directory after the load """ archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}.tgz") repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) loading_temp_directory = str(tmp_path / "loading") os.mkdir(loading_temp_directory) loader = SvnLoaderFromRemoteDump( swh_storage, repo_url, temp_directory=loading_temp_directory ) assert loader.load() == {"status": "eventful"} # the root temporary directory still exists assert os.path.exists(loader.temp_directory) # but it should be empty assert os.listdir(loader.temp_directory) == [] # the internal temp_dir should be cleaned up though assert not os.path.exists(loader.temp_dir) def test_loader_svn_cleanup_loader_from_dump_archive(swh_storage, datadir, tmp_path): """Loader should clean up its working directory after the load """ archive_ori_dump = os.path.join(datadir, "penguinsdbtools2018.dump.gz") archive_dump_dir = os.path.join(tmp_path, "dump") os.mkdir(archive_dump_dir) archive_dump = os.path.join(archive_dump_dir, "penguinsdbtools2018.dump.gz") # loader now drops the dump as soon as it's mounted so we need to make a copy first shutil.copyfile(archive_ori_dump, archive_dump) loading_path = str(tmp_path / "loading") os.mkdir(loading_path) # Prepare the dump as a local svn repository for test purposes temp_dir, repo_path = init_svn_repo_from_dump( archive_dump, root_dir=tmp_path, gzip=True ) repo_url = f"file://{repo_path}" loader = SvnLoaderFromRemoteDump(swh_storage, repo_url, temp_directory=loading_path) assert loader.load() == {"status": "eventful"} # the root temporary directory still exists assert os.path.exists(loader.temp_directory) # but it should be empty assert os.listdir(loader.temp_directory) == [] # the internal temp_dir should be cleaned up though assert not os.path.exists(loader.temp_dir) def test_svn_loader_from_remote_dump(swh_storage, datadir, tmp_path): archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}.tgz") repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) loaderFromDump = SvnLoaderFromRemoteDump( swh_storage, repo_url, temp_directory=tmp_path ) assert loaderFromDump.load() == {"status": "eventful"} assert_last_visit_matches( loaderFromDump.storage, repo_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, ) origin_url = repo_url + "2" # rename to another origin loader = SvnLoader( swh_storage, repo_url, origin_url=origin_url, temp_directory=tmp_path ) assert loader.load() == {"status": "eventful"} # because are working on new origin assert_last_visit_matches( loader.storage, origin_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, ) check_snapshot(GOURMET_SNAPSHOT, loader.storage) stats = get_stats(loader.storage) assert stats["origin"] == 2 # created one more origin assert stats["origin_visit"] == 2 assert stats["snapshot"] == 1 loader = SvnLoader( swh_storage, repo_url, temp_directory=tmp_path ) # no change on the origin-url assert loader.load() == {"status": "uneventful"} assert_last_visit_matches( loader.storage, origin_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, ) stats = get_stats(loader.storage) assert stats["origin"] == 2 assert stats["origin_visit"] == 3 assert stats["snapshot"] == 1 # second visit from the dump should be uneventful loaderFromDump = SvnLoaderFromRemoteDump( swh_storage, repo_url, temp_directory=tmp_path ) assert loaderFromDump.load() == {"status": "uneventful"} def test_svn_loader_from_remote_dump_incremental_load_on_stale_repo( swh_storage, datadir, tmp_path, mocker ): archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}.tgz") repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) # first load: a dump file will be created, mounted to a local repository # and the latter will be loaded into the archive loaderFromDump = SvnLoaderFromRemoteDump( swh_storage, repo_url, temp_directory=tmp_path ) assert loaderFromDump.load() == {"status": "eventful"} assert_last_visit_matches( loaderFromDump.storage, repo_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, ) # second load on same repository: the loader will detect there is no changes # since last load and will skip the dump, mount and load phases loaderFromDump = SvnLoaderFromRemoteDump( swh_storage, repo_url, temp_directory=tmp_path ) loaderFromDump.dump_svn_revisions = mocker.MagicMock() init_svn_repo_from_dump = mocker.patch( "swh.loader.svn.loader.init_svn_repo_from_dump" ) loaderFromDump.process_svn_revisions = mocker.MagicMock() loaderFromDump._check_revision_divergence = mocker.MagicMock() assert loaderFromDump.load() == {"status": "uneventful"} assert_last_visit_matches( loaderFromDump.storage, repo_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, ) # no dump loaderFromDump.dump_svn_revisions.assert_not_called() # no mount init_svn_repo_from_dump.assert_not_called() # no loading loaderFromDump.process_svn_revisions.assert_not_called() # no redundant post_load processing loaderFromDump._check_revision_divergence.assert_not_called() def test_svn_loader_from_remote_dump_incremental_load_on_non_stale_repo( swh_storage, datadir, tmp_path, mocker ): archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}.tgz") repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) # first load loader = SvnLoaderFromRemoteDump(swh_storage, repo_url, temp_directory=tmp_path) loader.load() archive_path = os.path.join(datadir, "pkg-gourmet-with-updates.tgz") repo_updated_url = prepare_repository_from_archive( archive_path, archive_name, tmp_path ) # second load loader = SvnLoaderFromRemoteDump( swh_storage, repo_updated_url, temp_directory=tmp_path ) dump_svn_revisions = mocker.spy(loader, "dump_svn_revisions") process_svn_revisions = mocker.spy(loader, "process_svn_revisions") loader.load() dump_svn_revisions.assert_called() process_svn_revisions.assert_called() def test_loader_user_defined_svn_properties(swh_storage, datadir, tmp_path): """Edge cases: The repository held some user defined svn-properties with special encodings, this prevented the repository from being loaded even though we do not ingest those information. """ archive_name = "httthttt" archive_path = os.path.join(datadir, f"{archive_name}.tgz") repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) loader = SvnLoader(swh_storage, repo_url) assert loader.load() == {"status": "eventful"} expected_snapshot = Snapshot( id=hash_to_bytes("70487267f682c07e52a2371061369b6cf5bffa47"), branches={ b"HEAD": SnapshotBranch( target=hash_to_bytes("604a17dbb15e8d7ecb3e9f3768d09bf493667a93"), target_type=TargetType.REVISION, ) }, ) check_snapshot(expected_snapshot, loader.storage) assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", snapshot=expected_snapshot.id, ) stats = get_stats(loader.storage) assert stats["origin"] == 1 assert stats["origin_visit"] == 1 assert stats["snapshot"] == 1 assert stats["revision"] == 7 def test_loader_svn_dir_added_then_removed(swh_storage, datadir, tmp_path): """Loader should handle directory removal when processing a commit""" archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}-add-remove-dir.tgz") repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) assert loader.load() == {"status": "eventful"} assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", ) def test_loader_svn_loader_from_dump_archive(swh_storage, datadir, tmp_path): archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}.tgz") repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) origin_url = f"svn://{archive_name}" dump_filename = f"{archive_name}.dump" with open(os.path.join(tmp_path, dump_filename), "wb") as dump_file: # create compressed dump file of pkg-gourmet repo subprocess.run(["svnrdump", "dump", repo_url], stdout=dump_file) subprocess.run(["gzip", dump_filename], cwd=tmp_path) # load svn repo from that compressed dump file loader = SvnLoaderFromDumpArchive( swh_storage, url=origin_url, archive_path=os.path.join(tmp_path, f"{dump_filename}.gz"), temp_directory=tmp_path, ) assert loader.load() == {"status": "eventful"} assert_last_visit_matches( loader.storage, origin_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, ) check_snapshot(GOURMET_SNAPSHOT, loader.storage) assert get_stats(loader.storage) == { "content": 19, "directory": 17, "origin": 1, "origin_visit": 1, "release": 0, "revision": 6, "skipped_content": 0, "snapshot": 1, } class CommitChangeType(Enum): AddOrUpdate = 1 Delete = 2 class CommitChange(TypedDict, total=False): change_type: CommitChangeType path: str properties: Dict[str, str] data: bytes def add_commit(repo_url: str, message: str, changes: List[CommitChange]) -> None: conn = RemoteAccess(repo_url, auth=Auth([get_username_provider()])) editor = conn.get_commit_editor({"svn:log": message}) root = editor.open_root() for change in changes: if change["change_type"] == CommitChangeType.Delete: root.delete_entry(change["path"].rstrip("/")) else: dir_change = change["path"].endswith("/") split_path = change["path"].rstrip("/").split("/") for i in range(len(split_path)): path = "/".join(split_path[0 : i + 1]) if i < len(split_path) - 1: try: root.add_directory(path).close() except SubversionException: pass else: if dir_change: root.add_directory(path).close() else: try: file = root.add_file(path) except SubversionException: file = root.open_file(path) if "properties" in change: for prop, value in change["properties"].items(): file.change_prop(prop, value) if "data" in change: txdelta = file.apply_textdelta() delta.send_stream(BytesIO(change["data"]), txdelta) file.close() root.close() editor.close() def test_loader_eol_style_file_property_handling_edge_case(swh_storage, tmp_path): # create a repository repo_path = os.path.join(tmp_path, "tmprepo") repos.create(repo_path) repo_url = f"file://{repo_path}" # # first commit add_commit( repo_url, ( "Add a directory containing a file with CRLF end of line " "and set svn:eol-style property to native so CRLF will be " "replaced by LF in the file when exporting the revision" ), [ CommitChange( change_type=CommitChangeType.AddOrUpdate, path="directory/file_with_crlf_eol.txt", properties={"svn:eol-style": "native"}, data=b"Hello world!\r\n", ) ], ) # second commit add_commit( repo_url, "Remove previously added directory and file", [CommitChange(change_type=CommitChangeType.Delete, path="directory/",)], ) # third commit add_commit( repo_url, ( "Add again same directory containing same file with CRLF end of line " "but do not set svn:eol-style property value so CRLF will not be " "replaced by LF when exporting the revision" ), [ CommitChange( change_type=CommitChangeType.AddOrUpdate, path="directory/file_with_crlf_eol.txt", data=b"Hello world!\r\n", ) ], ) # instantiate a svn loader checking after each processed revision that # the repository filesystem it reconstructed does not differ from a subversion # export of that revision loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) assert loader.load() == {"status": "eventful"} assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", ) assert get_stats(loader.storage) == { "content": 2, "directory": 5, "origin": 1, "origin_visit": 1, "release": 0, "revision": 3, "skipped_content": 0, "snapshot": 1, } def get_head_revision_paths_info(loader: SvnLoader) -> Dict[bytes, Dict[str, Any]]: assert loader.snapshot is not None root_dir = loader.snapshot.branches[b"HEAD"].target revision = loader.storage.revision_get([root_dir])[0] assert revision is not None paths = {} for entry in loader.storage.directory_ls(revision.directory, recursive=True): paths[entry["name"]] = entry return paths def test_loader_eol_style_on_svn_link_handling(swh_storage, tmp_path): # create a repository repo_path = os.path.join(tmp_path, "tmprepo") repos.create(repo_path) repo_url = f"file://{repo_path}" # first commit add_commit( repo_url, ( "Add a regular file, a directory and a link to the regular file " "in the directory. Set svn:eol-style property for the regular " "file and the link. Set svn:special property for the link." ), [ CommitChange( change_type=CommitChangeType.AddOrUpdate, path="file_with_crlf_eol.txt", properties={"svn:eol-style": "native"}, data=b"Hello world!\r\n", ), CommitChange( change_type=CommitChangeType.AddOrUpdate, path="directory/file_with_crlf_eol.txt", properties={"svn:eol-style": "native", "svn:special": "*"}, data=b"link ../file_with_crlf_eol.txt", ), ], ) # instantiate a svn loader checking after each processed revision that # the repository filesystem it reconstructed does not differ from a subversion # export of that revision loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) assert loader.load() == {"status": "eventful"} assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", ) # check loaded objects are those expected assert get_stats(loader.storage) == { "content": 2, "directory": 2, "origin": 1, "origin_visit": 1, "release": 0, "revision": 1, "skipped_content": 0, "snapshot": 1, } paths = get_head_revision_paths_info(loader) assert ( loader.storage.content_get_data(paths[b"file_with_crlf_eol.txt"]["sha1"]) == b"Hello world!\n" ) assert paths[b"directory/file_with_crlf_eol.txt"]["perms"] == DentryPerms.symlink assert ( loader.storage.content_get_data( paths[b"directory/file_with_crlf_eol.txt"]["sha1"] ) == b"../file_with_crlf_eol.txt" ) def test_loader_svn_special_property_unset(swh_storage, tmp_path): # create a repository repo_path = os.path.join(tmp_path, "tmprepo") repos.create(repo_path) repo_url = f"file://{repo_path}" # first commit add_commit( repo_url, ( "Create a regular file, a link to a file and a link to an " "external file. Set the svn:special property on the links." ), [ CommitChange( change_type=CommitChangeType.AddOrUpdate, path="file.txt", data=b"Hello world!\n", ), CommitChange( change_type=CommitChangeType.AddOrUpdate, path="link.txt", properties={"svn:special": "*"}, data=b"link ./file.txt", ), CommitChange( change_type=CommitChangeType.AddOrUpdate, path="external_link.txt", properties={"svn:special": "*"}, data=b"link /home/user/data.txt", ), ], ) # second commit add_commit( repo_url, "Unset the svn:special property on the links.", [ CommitChange( change_type=CommitChangeType.AddOrUpdate, path="link.txt", properties={"svn:special": None}, ), CommitChange( change_type=CommitChangeType.AddOrUpdate, path="external_link.txt", properties={"svn:special": None}, ), ], ) # instantiate a svn loader checking after each processed revision that # the repository filesystem it reconstructed does not differ from a subversion # export of that revision loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) assert loader.load() == {"status": "eventful"} assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", ) # check loaded objects are those expected assert get_stats(loader.storage) == { "content": 5, "directory": 2, "origin": 1, "origin_visit": 1, "release": 0, "revision": 2, "skipped_content": 0, "snapshot": 1, } paths = get_head_revision_paths_info(loader) assert paths[b"link.txt"]["perms"] == DentryPerms.content assert ( loader.storage.content_get_data(paths[b"link.txt"]["sha1"]) == b"link ./file.txt" ) assert paths[b"external_link.txt"]["perms"] == DentryPerms.content assert ( loader.storage.content_get_data(paths[b"external_link.txt"]["sha1"]) == b"link /home/user/data.txt" ) def test_loader_invalid_svn_eol_style_property_value(swh_storage, tmp_path): # create a repository repo_path = os.path.join(tmp_path, "tmprepo") repos.create(repo_path) repo_url = f"file://{repo_path}" filename = "file_with_crlf_eol.txt" file_content = b"Hello world!\r\n" # # first commit add_commit( repo_url, ( "Add a file with CRLF end of line and set svn:eol-style property " "to an invalid value." ), [ CommitChange( change_type=CommitChangeType.AddOrUpdate, path=filename, properties={"svn:eol-style": "foo"}, data=file_content, ) ], ) # instantiate a svn loader checking after each processed revision that # the repository filesystem it reconstructed does not differ from a subversion # export of that revision loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) assert loader.load() == {"status": "eventful"} assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", ) paths = get_head_revision_paths_info(loader) # end of lines should not have been processed assert ( loader.storage.content_get_data(paths[filename.encode()]["sha1"]) == file_content ) def test_loader_first_revision_is_not_number_one(swh_storage, mocker, tmp_path): class SvnRepoSkipFirstRevision(SvnRepo): def logs(self, revision_start, revision_end): """Overrides logs method to skip revision number one in yielded revisions""" yield from super().logs(revision_start + 1, revision_end) from swh.loader.svn import loader mocker.patch.object(loader, "SvnRepo", SvnRepoSkipFirstRevision) # create a repository repo_path = os.path.join(tmp_path, "tmprepo") repos.create(repo_path) repo_url = f"file://{repo_path}" for filename in ("foo", "bar", "baz"): add_commit( repo_url, f"Add {filename} file", [ CommitChange( change_type=CommitChangeType.AddOrUpdate, path=filename, data=f"{filename}\n".encode(), ) ], ) loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) # post loading will detect an issue and make a partial visit with a snapshot assert loader.load() == {"status": "failed"} assert_last_visit_matches( loader.storage, repo_url, status="partial", type="svn", ) assert get_stats(loader.storage) == { "content": 2, "directory": 2, "origin": 1, "origin_visit": 1, "release": 0, "revision": 2, "skipped_content": 0, "snapshot": 1, } def test_loader_svn_special_property_on_binary_file_with_null_byte( swh_storage, tmp_path ): """When a file has the svn:special property set but is not a svn link, it will be truncated when performing an export operation if it contains a null byte. Indeed, subversion will treat the file content as text but it might be a binary file containing null bytes.""" # create a repository repo_path = os.path.join(tmp_path, "tmprepo") repos.create(repo_path) repo_url = f"file://{repo_path}" data = ( b"!\xff\xfea\x00p\x00t\x00-\x00c\x00y\x00g\x00.\x00s\x00h\x00\x00\x00" ) # first commit add_commit( repo_url, "Add a non svn link binary file and set the svn:special property on it", [ CommitChange( change_type=CommitChangeType.AddOrUpdate, path="binary_file", properties={"svn:special": "*"}, data=data, ), ], ) # second commit add_commit( repo_url, "Remove the svn:special property on the previously added file", [ CommitChange( change_type=CommitChangeType.AddOrUpdate, path="binary_file", properties={"svn:special": None}, ), ], ) # instantiate a svn loader checking after each processed revision that # the repository filesystem it reconstructed does not differ from a subversion # export of that revision loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) assert loader.load() == {"status": "eventful"} assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", ) def test_loader_last_revision_divergence(swh_storage, datadir, tmp_path): archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}.tgz") repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) class SvnLoaderRevisionDivergence(SvnLoader): def _check_revision_divergence(self, count, rev, dir_id): raise ValueError("revision divergence detected") loader = SvnLoaderRevisionDivergence(swh_storage, repo_url, temp_directory=tmp_path) assert loader.load() == {"status": "failed"} assert_last_visit_matches( loader.storage, repo_url, status="partial", type="svn", snapshot=GOURMET_SNAPSHOT.id, ) def test_loader_delete_directory_while_file_has_same_prefix(swh_storage, tmp_path): # create a repository repo_path = os.path.join(tmp_path, "tmprepo") repos.create(repo_path) repo_url = f"file://{repo_path}" # first commit add_commit( repo_url, "Add a file and a directory with same prefix", [ CommitChange( change_type=CommitChangeType.AddOrUpdate, path="foo/bar.c", data=b'#include "../foo.c"', ), CommitChange( change_type=CommitChangeType.AddOrUpdate, path="foo.c", data=b"int foo() {return 0;}", ), ], ) # second commit add_commit( repo_url, "Delete previously added directory and update file content", [ CommitChange(change_type=CommitChangeType.Delete, path="foo"), CommitChange( change_type=CommitChangeType.AddOrUpdate, path="foo.c", data=b"int foo() {return 1;}", ), ], ) # instantiate a svn loader checking after each processed revision that # the repository filesystem it reconstructed does not differ from a subversion # export of that revision loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) assert loader.load() == {"status": "eventful"} assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", ) def test_svn_loader_incremental(swh_storage, tmp_path): # create a repository repo_path = os.path.join(tmp_path, "tmprepo") repos.create(repo_path) repo_url = f"file://{repo_path}" # first commit add_commit( repo_url, ( "Add a directory containing a file with CRLF end of line " "and set svn:eol-style property to native so CRLF will be " "replaced by LF in the file when exporting the revision" ), [ CommitChange( change_type=CommitChangeType.AddOrUpdate, path="file_with_crlf_eol.txt", properties={"svn:eol-style": "native"}, data=b"Hello world!\r\n", ) ], ) # first load loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) assert loader.load() == {"status": "eventful"} assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", ) # second commit add_commit( repo_url, "Modify previously added file", [ CommitChange( change_type=CommitChangeType.AddOrUpdate, path="file_with_crlf_eol.txt", data=b"Hello World!\r\n", ) ], ) # second load, incremental loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) assert loader.load() == {"status": "eventful"} assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", ) # third commit add_commit( repo_url, "Unset svn:eol-style property on file", [ CommitChange( change_type=CommitChangeType.AddOrUpdate, path="file_with_crlf_eol.txt", properties={"svn:eol-style": None}, ) ], ) # third load, incremental loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) assert loader.load() == {"status": "eventful"} assert_last_visit_matches( loader.storage, repo_url, status="full", type="svn", ) + + +def test_svn_loader_incremental_replay_start_with_empty_directory( + swh_storage, mocker, tmp_path +): + # create a repository + repo_path = os.path.join(tmp_path, "tmprepo") + repos.create(repo_path) + repo_url = f"file://{repo_path}" + + # first commit + add_commit( + repo_url, + ("Add a file"), + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, path="foo.txt", data=b"foo\n", + ) + ], + ) + + # first load + loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + + # second commit + add_commit( + repo_url, + "Modify previously added file", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, path="foo.txt", data=b"bar\n", + ) + ], + ) + + class SvnRepoCheckReplayStartWithEmptyDirectory(SvnRepo): + def swh_hash_data_per_revision(self, start_revision: int, end_revision: int): + """Overrides swh_hash_data_per_revision method to grab the content + of the directory where the svn revisions will be replayed before that + process starts.""" + self.replay_dir_content_before_start = [ + os.path.join(root, name) + for root, _, files in os.walk(self.local_url) + for name in files + ] + yield from super().swh_hash_data_per_revision(start_revision, end_revision) + + from swh.loader.svn import loader + + mocker.patch.object(loader, "SvnRepo", SvnRepoCheckReplayStartWithEmptyDirectory) + + # second load, incremental + loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) + loader.load() + + # check work directory was empty before replaying revisions + assert loader.svnrepo.replay_dir_content_before_start == [] diff --git a/swh/loader/svn/tests/test_task.py b/swh/loader/svn/tests/test_task.py index 1911e39..f1dbcfa 100644 --- a/swh/loader/svn/tests/test_task.py +++ b/swh/loader/svn/tests/test_task.py @@ -1,55 +1,76 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from datetime import datetime, timezone + +import pytest + +from swh.loader.svn.tasks import convert_to_datetime + def test_svn_loader( mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config ): mock_loader = mocker.patch("swh.loader.svn.loader.SvnLoader.load") mock_loader.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.svn.tasks.LoadSvnRepository", kwargs=dict(url="some-technical-url", origin_url="origin-url"), ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"} def test_svn_loader_from_dump( mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config ): mock_loader = mocker.patch("swh.loader.svn.loader.SvnLoaderFromDumpArchive.load") mock_loader.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.svn.tasks.MountAndLoadSvnRepository", kwargs=dict(url="some-url", archive_path="some-path"), ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"} def test_svn_loader_from_remote_dump( mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config ): mock_loader = mocker.patch("swh.loader.svn.loader.SvnLoaderFromRemoteDump.load") mock_loader.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.svn.tasks.DumpMountAndLoadSvnRepository", kwargs=dict(url="some-remote-dump-url", origin_url="origin-url"), ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"} + + +@pytest.mark.parametrize( + "date,expected_result", + [ + (None, None), + ( + "2021-11-23 09:41:02.434195+00:00", + datetime(2021, 11, 23, 9, 41, 2, 434195, tzinfo=timezone.utc), + ), + ("23112021", None,), # failure to parse + ], +) +def test_convert_to_datetime(date, expected_result): + assert convert_to_datetime(date) == expected_result