diff --git a/PKG-INFO b/PKG-INFO index f6876cb..f0ba53c 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,106 +1,106 @@ Metadata-Version: 2.1 Name: swh.loader.git -Version: 1.1.2 +Version: 1.1.3 Summary: Software Heritage git loader Home-page: https://forge.softwareheritage.org/diffusion/DLDG/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-loader-git Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-loader-git/ Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing License-File: LICENSE License-File: AUTHORS swh-loader-git ============== The Software Heritage Git Loader is a tool and a library to walk a local Git repository and inject into the SWH dataset all contained files that weren't known before. The main entry points are - :class:`swh.loader.git.loader.GitLoader` for the main loader which ingests a remote git repository's contents. - :class:`swh.loader.git.from_disk.GitLoaderFromDisk` which ingests a local git clone repository. - :class:`swh.loader.git.loader.GitLoaderFromArchive` which ingests a git repository wrapped in an archive. License ------- This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. See top-level LICENSE file for the full text of the GNU General Public License along with this program. Dependencies ------------ ### Runtime - python3 - python3-dulwich - python3-retrying - python3-swh.core - python3-swh.model - python3-swh.storage - python3-swh.scheduler ### Test - python3-nose Requirements ------------ - implementation language, Python3 - coding guidelines: conform to PEP8 - Git access: via dulwich CLI Run ---------- You can run the loader from a remote origin (*loader*) or from an origin on disk (*from_disk*) directly by calling: ``` swh loader -C run git ``` or "git_disk". ## Configuration sample /tmp/git.yml: ``` storage: cls: remote args: url: http://localhost:5002/ ``` diff --git a/debian/changelog b/debian/changelog index d06b5a0..5084622 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,723 +1,726 @@ -swh-loader-git (1.1.2-1~swh1~bpo10+1) buster-swh; urgency=medium +swh-loader-git (1.1.3-1~swh1) unstable-swh; urgency=medium - * Rebuild for buster-swh + * New upstream release 1.1.3 - (tagged by Valentin Lorentz + on 2021-10-18 14:21:05 +0200) + * Upstream changes: - v1.1.3 - * converters: Prevent zero + dates from being converted to null dates. - -- Software Heritage autobuilder (on jenkins-debian1) Tue, 05 Oct 2021 16:03:10 +0000 + -- Software Heritage autobuilder (on jenkins-debian1) Mon, 18 Oct 2021 12:23:52 +0000 swh-loader-git (1.1.2-1~swh1) unstable-swh; urgency=medium * New upstream release 1.1.2 - (tagged by Antoine Lambert on 2021-10-05 17:46:41 +0200) * Upstream changes: - version 1.1.2 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 05 Oct 2021 16:01:27 +0000 swh-loader-git (1.1.1-1~swh1) unstable-swh; urgency=medium * New upstream release 1.1.1 - (tagged by Antoine R. Dumont (@ardumont) on 2021-09-30 09:30:46 +0200) * Upstream changes: - v1.1.1 - Use correct logging instruction, let log.info format entries - Clarify local/remote heads type as those are hexadecimal bytes str -- Software Heritage autobuilder (on jenkins-debian1) Thu, 30 Sep 2021 07:33:48 +0000 swh-loader-git (1.1.0-1~swh1) unstable-swh; urgency=medium * New upstream release 1.1.0 - (tagged by Antoine Lambert on 2021-09-28 15:10:11 +0200) * Upstream changes: - version 1.1.0 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 28 Sep 2021 13:14:10 +0000 swh-loader-git (1.0.1-1~swh1) unstable-swh; urgency=medium * New upstream release 1.0.1 - (tagged by Antoine R. Dumont (@ardumont) on 2021-09-22 09:51:39 +0200) * Upstream changes: - v1.0.1 - Fix tests for Dulwich < 0.20.22 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 22 Sep 2021 07:54:07 +0000 swh-loader-git (1.0.0-1~swh1) unstable-swh; urgency=medium * New upstream release 1.0.0 - (tagged by Valentin Lorentz on 2021-09-17 12:28:36 +0200) * Upstream changes: - v1.0.0 - * from_disk: Do not drop tags with missing tagger or date - * Migrate to pytest-style tests - * converters: Recompute hashes and check they match the originals -- Software Heritage autobuilder (on jenkins-debian1) Fri, 17 Sep 2021 10:31:49 +0000 swh-loader-git (0.10.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.10.1 - (tagged by Valentin Lorentz on 2021-08-03 10:37:43 +0200) * Upstream changes: - v0.10.1 - * from_disk: Improve error logging - * converters: Preserve GPG signatures on releases - * Do not exclude falsy git objects from being added. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 03 Aug 2021 08:46:18 +0000 swh-loader-git (0.10.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.10.0 - (tagged by Antoine R. Dumont (@ardumont) on 2021-06-08 10:23:41 +0200) * Upstream changes: - v0.10.0 - Spool large packfiles to disk instead of consuming tons of memory -- Software Heritage autobuilder (on jenkins-debian1) Tue, 08 Jun 2021 08:27:09 +0000 swh-loader-git (0.9.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.9.1 - (tagged by Antoine R. Dumont (@ardumont) on 2021-04-16 15:36:06 +0200) * Upstream changes: - v0.9.1 - Fix Pack File too big error formatting - Rename 'git_metadata' to 'extra_headers' -- Software Heritage autobuilder (on jenkins-debian1) Fri, 16 Apr 2021 13:39:09 +0000 swh-loader-git (0.9.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.9.0 - (tagged by Nicolas Dandrimont on 2021-02-25 18:44:39 +0100) * Upstream changes: - Release swh.loader.git 0.9.0 - Throughput and memory usage improvements -- Software Heritage autobuilder (on jenkins-debian1) Thu, 25 Feb 2021 17:52:31 +0000 swh-loader-git (0.8.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.8.0 - (tagged by Antoine R. Dumont (@ardumont) on 2021-02-17 15:07:34 +0100) * Upstream changes: - v0.8.0 - Rework loader instantiation logic according to loader core api -- Software Heritage autobuilder (on jenkins-debian1) Wed, 17 Feb 2021 14:10:13 +0000 swh-loader-git (0.7.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.7.0 - (tagged by Antoine R. Dumont (@ardumont) on 2021-02-12 17:31:15 +0100) * Upstream changes: - v0.7.0 - loader.git: Mark visit status as not_found or failed when relevant - loader.git: Explicit the failure test cases -- Software Heritage autobuilder (on jenkins-debian1) Fri, 12 Feb 2021 16:33:18 +0000 swh-loader-git (0.6.0-1~swh2) unstable-swh; urgency=medium * Bump dependencies -- Antoine R. Dumont (@ardumont) Wed, 03 Feb 2021 14:46:51 +0100 swh-loader-git (0.6.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.6.0 - (tagged by Antoine R. Dumont (@ardumont) on 2021-02-03 14:31:51 +0100) * Upstream changes: - v0.6.0 - Adapt origin_get_latest_visit_status according to latest api change - tox.ini: Add swh.core[testing] requirement - from_disk: Fix mypy error with dulwich >= 0.20.13 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 03 Feb 2021 13:34:21 +0000 swh-loader-git (0.5.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.5.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-11-13 10:34:02 +0100) * Upstream changes: - v0.5.0 - loader.git.from_disk: Register loader in `swh loader run` cli -- Software Heritage autobuilder (on jenkins-debian1) Fri, 13 Nov 2020 09:34:28 +0000 swh-loader-git (0.4.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.4.1 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-02 17:07:58 +0200) * Upstream changes: - v0.4.1 - git.loader*: Open configuration passing from constructor - tox.ini: pin black to the pre-commit version (19.10b0) to avoid flip-flops -- Software Heritage autobuilder (on jenkins-debian1) Fri, 02 Oct 2020 15:08:42 +0000 swh-loader-git (0.4.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.4.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-02 14:19:17 +0200) * Upstream changes: - v0.4.0 - git.loader: Migrate away from SWHConfig mixin - Drop vcversioner from setup.py (superseded by setuptools-scm) - tests: Don't check the number of created 'person' objects. - python: Reorder imports with isort - pre- commit: Add isort hook and configuration - pre-commit: Update flake8 hook configuration - Tell pytest not to recurse in dotdirs. - tests: Replace calls to snapshot_get with snapshot_get_all_branches. -- Software Heritage autobuilder (on jenkins-debian1) Fri, 02 Oct 2020 12:19:50 +0000 swh-loader-git (0.3.6-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.6 - (tagged by Antoine R. Dumont (@ardumont) on 2020-08-06 16:49:04 +0200) * Upstream changes: - v0.3.6 - Adapt code according to storage signature -- Software Heritage autobuilder (on jenkins-debian1) Thu, 06 Aug 2020 14:50:22 +0000 swh-loader-git (0.3.5-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.5 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-28 07:49:13 +0200) * Upstream changes: - v0.3.5 - loader: Update swh.storage.origin_get call to latest api change -- Software Heritage autobuilder (on jenkins-debian1) Tue, 28 Jul 2020 05:51:16 +0000 swh-loader-git (0.3.4-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.4 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-26 05:55:34 +0200) * Upstream changes: - v0.3.4 - setup.py: Migrate from vcversioner to setuptools-scm - MANIFEST: Add missing conftest requirement -- Software Heritage autobuilder (on jenkins-debian1) Sun, 26 Jul 2020 03:57:36 +0000 swh-loader-git (0.3.3-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.3 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-17 15:29:21 +0200) * Upstream changes: - v0.3.3 - tests: Reuse pytest fixtures from swh.loader.core - tests: Check against snapshot model object -- Software Heritage autobuilder (on jenkins-debian1) Fri, 17 Jul 2020 13:30:55 +0000 swh-loader-git (0.3.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.2 - (tagged by David Douard on 2020-07-16 10:46:20 +0200) * Upstream changes: - v0.3.2 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 16 Jul 2020 08:49:36 +0000 swh-loader-git (0.3.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.1 - (tagged by Antoine Lambert on 2020-07-08 16:37:09 +0200) * Upstream changes: - version 0.3.1 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 08 Jul 2020 14:39:48 +0000 swh-loader-git (0.3.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.0 - (tagged by David Douard on 2020-07-08 15:35:34 +0200) * Upstream changes: - v0.3.0 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 08 Jul 2020 13:38:18 +0000 swh-loader-git (0.2.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-06-23 15:14:13 +0200) * Upstream changes: - v0.2.0 - loader: Read snapshot out of the last origin visit status - tests: Use assert_last_visit_matches - Adapt visit_date type from string to datetime -- Software Heritage autobuilder (on jenkins-debian1) Tue, 23 Jun 2020 13:15:36 +0000 swh-loader-git (0.1.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.1.2 - (tagged by Antoine Lambert on 2020-06-03 14:54:36 +0200) * Upstream changes: - version 0.1.2 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 03 Jun 2020 12:57:56 +0000 swh-loader-git (0.1.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.1.1 - (tagged by Antoine Lambert on 2020-06-02 17:44:44 +0200) * Upstream changes: - version 0.1.1 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 02 Jun 2020 15:50:08 +0000 swh-loader-git (0.1.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.1.0 - (tagged by Nicolas Dandrimont on 2020-05-29 10:33:12 +0200) * Upstream changes: - Release swh.loader.git v0.1.0 - Use the previous snapshot instead of any object from the archive to do - incremental loads - Merge branch filtering behavior between the local and remote loaders - Add default target branch for symbolic references -- Software Heritage autobuilder (on jenkins-debian1) Fri, 29 May 2020 08:37:48 +0000 swh-loader-git (0.0.60-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.60 - (tagged by Antoine R. Dumont (@ardumont) on 2020-04-15 11:52:55 +0200) * Upstream changes: - v0.0.60 - git.loader: fix failing origin visit update step - Add a pyproject.toml file to target py37 for black - Enable black -- Software Heritage autobuilder (on jenkins-debian1) Wed, 15 Apr 2020 10:05:51 +0000 swh-loader-git (0.0.59-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.59 - (tagged by Antoine Lambert on 2020-04-06 11:59:27 +0200) * Upstream changes: - version 0.0.59 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 06 Apr 2020 10:04:59 +0000 swh-loader-git (0.0.58-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.58 - (tagged by Valentin Lorentz on 2020-03-02 11:25:43 +0100) * Upstream changes: - v0.0.58 - * Use origin_visit_get_latest instead of snapshot_get_latest. - * Use swh-model objects instead of dicts. -- Software Heritage autobuilder (on jenkins-debian1) Mon, 02 Mar 2020 10:28:37 +0000 swh-loader-git (0.0.57-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.57 - (tagged by Antoine R. Dumont (@ardumont) on 2020-02-07 03:32:49 +0100) * Upstream changes: - v0.0.57 - loaders: Remove content size computation during conversion -- Software Heritage autobuilder (on jenkins-debian1) Fri, 07 Feb 2020 02:45:56 +0000 swh-loader-git (0.0.56-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.56 - (tagged by Antoine R. Dumont (@ardumont) on 2020-01-28 13:24:24 +0100) * Upstream changes: - v0.0.56 - git.loader: Migrate from UnbufferedLoader to DVCSLoader -- Software Heritage autobuilder (on jenkins-debian1) Tue, 28 Jan 2020 12:27:02 +0000 swh-loader-git (0.0.55-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.55 - (tagged by Antoine R. Dumont (@ardumont) on 2019-12-12 14:41:10 +0100) * Upstream changes: - v0.0.55 - loader: Bump dependency on loader-core -- Software Heritage autobuilder (on jenkins-debian1) Thu, 12 Dec 2019 13:44:22 +0000 swh-loader-git (0.0.54-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.54 - (tagged by Antoine R. Dumont (@ardumont) on 2019-12-12 11:43:50 +0100) * Upstream changes: - v0.0.54 - tasks: Enforce kwargs use in task message -- Software Heritage autobuilder (on jenkins-debian1) Thu, 12 Dec 2019 10:46:28 +0000 swh-loader-git (0.0.53-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.53 - (tagged by Antoine R. Dumont (@ardumont) on 2019-12-10 11:24:30 +0100) * Upstream changes: - v0.0.53 - tasks: Unify message format with other loaders - tasks: Use celery's shared_task decorator - tests: Migrate to pytest-mock's fixture - loader.git: Register git worker - tasks: Rename task according to production - git: Unify loaders constructor - Fix a typo reported by codespell - Add a pre-commit config file - Migrate tox.ini to extras = xxx instead of deps = .[testing] - De-specify testenv:py3 - Drop version constraint on pytest < 4 - Include all requirements in MANIFEST.in - Add support for symbolic references -- Software Heritage autobuilder (on jenkins-debian1) Tue, 10 Dec 2019 10:27:32 +0000 swh-loader-git (0.0.52-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.52 - (tagged by Stefano Zacchiroli on 2019-10-10 12:07:05 +0200) * Upstream changes: - v0.0.52 - (brown paper bag release) - * MANIFEST.in: ship py.typed -- Software Heritage autobuilder (on jenkins-debian1) Thu, 10 Oct 2019 10:12:13 +0000 swh-loader-git (0.0.51-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.51 - (tagged by Stefano Zacchiroli on 2019-10-10 11:59:01 +0200) * Upstream changes: - v0.0.51 - * tox.ini: Fix py3 environment to use packaged tests - * typing: minimal changes to make a no- op mypy run pass - * test_from_disk.py: avoid shadowing base classes in tests -- Software Heritage autobuilder (on jenkins-debian1) Thu, 10 Oct 2019 10:02:08 +0000 swh-loader-git (0.0.50-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.50 - (tagged by Antoine Lambert on 2019-09-03 13:07:54 +0200) * Upstream changes: - version 0.0.50 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 03 Sep 2019 11:13:19 +0000 swh-loader-git (0.0.49-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.49 - (tagged by Valentin Lorentz on 2019-06-12 15:05:10 +0200) * Upstream changes: - Use origin URLs instead of numeric ids. -- Software Heritage autobuilder (on jenkins-debian1) Wed, 19 Jun 2019 10:28:05 +0000 swh-loader-git (0.0.48-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.48 - (tagged by Antoine R. Dumont (@ardumont) on 2019-01-30 11:18:55 +0100) * Upstream changes: - v0.0.48 - Bump dependency on swh- scheduler 0.0.39 - Rewrite celery tasks as a decorated function -- Software Heritage autobuilder (on jenkins-debian1) Wed, 30 Jan 2019 10:22:22 +0000 swh-loader-git (0.0.43-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.43 * Support the new paginated snapshot branch fetching functions -- Nicolas Dandrimont Thu, 18 Oct 2018 18:49:26 +0200 swh-loader-git (0.0.42-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.42 * Fix critical bug in incremental loading -- Nicolas Dandrimont Thu, 11 Oct 2018 17:19:07 +0200 swh-loader-git (0.0.41-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.41 * Use explicit keyword argument for base_url in the load task -- Nicolas Dandrimont Thu, 11 Oct 2018 16:26:27 +0200 swh-loader-git (0.0.40-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.40 * Improve python packaging * Make the loader more robust against holes in the history caused by * buggy imports * Allow ignoring the history to make a full load -- Nicolas Dandrimont Tue, 09 Oct 2018 16:28:14 +0200 swh-loader-git (0.0.39-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.39 * Avoid walking the history of large git repos, which takes a long time * Really save packfiles -- Nicolas Dandrimont Thu, 20 Sep 2018 17:22:17 +0200 swh-loader-git (0.0.38-1~swh1) unstable-swh; urgency=medium * v0.0.38 * Improve origin_visit initialization step * Properly sandbox the prepare statement so that if it breaks, we can * update appropriately the visit with the correct status -- Antoine R. Dumont (@ardumont) Wed, 07 Mar 2018 11:39:30 +0100 swh-loader-git (0.0.37-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.37 * Remove spurious debug print -- Nicolas Dandrimont Tue, 06 Feb 2018 16:00:40 +0100 swh-loader-git (0.0.36-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.36 * Update to use snapshots instead of occurrences * Use dulwich get_transport_and_path rather than hardcode the tcp transport -- Nicolas Dandrimont Tue, 06 Feb 2018 14:42:36 +0100 swh-loader-git (0.0.35-1~swh1) unstable-swh; urgency=medium * v0.0.35 * swh.loader.git.loader: Warn when object is corrupted and continue * swh.loader.git.loader: Add structured data to the log message regarding skipping objects * swh.loader.git.loader: Force further checks on objects * swh.loader.git.loader: Unify reading object from the repository * swh.loader.git.loader: Warn when object malformed and continue * swh.loader.git.loader: Trap missing object id and continue * swh.loader.git.base: Reuse swh.loader.core base loader * swh.loader.git.converters: Fix release time conversion issue when no date provided -- Antoine R. Dumont (@ardumont) Mon, 18 Dec 2017 12:08:01 +0100 swh-loader-git (0.0.34-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git version 0.0.34 * Update packaging runes -- Nicolas Dandrimont Thu, 12 Oct 2017 20:12:11 +0200 swh-loader-git (0.0.33-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.33 * make the updater's parent commit cache more useful -- Nicolas Dandrimont Fri, 15 Sep 2017 18:45:41 +0200 swh-loader-git (0.0.32-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git 0.0.32 * Update tasks to new swh.scheduler API -- Nicolas Dandrimont Mon, 12 Jun 2017 18:04:50 +0200 swh-loader-git (0.0.31-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.31 * Migrate from swh.core.hashutil to swh.model.hashutil * Only send objects that are actually missing -- Nicolas Dandrimont Fri, 17 Mar 2017 17:40:17 +0100 swh-loader-git (0.0.30-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.30 * Fix handling of mergetag headers -- Nicolas Dandrimont Thu, 09 Mar 2017 11:30:08 +0100 swh-loader-git (0.0.29-1~swh1) unstable-swh; urgency=medium * v0.0.29 * GitLoaderFromArchive: Use the same configuration file as * GitLoader (permit to deploy both as the same unit) * git reader: Refactor to allow listing revisions as well as contents -- Antoine R. Dumont (@ardumont) Mon, 20 Feb 2017 11:32:24 +0100 swh-loader-git (0.0.28-1~swh1) unstable-swh; urgency=medium * v0.0.28 * loader: Fix fetch_date override -- Antoine R. Dumont (@ardumont) Wed, 15 Feb 2017 18:43:32 +0100 swh-loader-git (0.0.27-1~swh1) unstable-swh; urgency=medium * v0.0.27 * Add loader-git from archive -- Antoine R. Dumont (@ardumont) Tue, 14 Feb 2017 18:56:52 +0100 swh-loader-git (0.0.26-1~swh1) unstable-swh; urgency=medium * v0.0.26 * Add a git loader able to deal with git repository in archive -- Antoine R. Dumont (@ardumont) Tue, 14 Feb 2017 16:24:50 +0100 swh-loader-git (0.0.25-1~swh1) unstable-swh; urgency=medium * v0.0.25 * Fix to permit to actually pass the fetch date as parameter for * the loading git disk loader -- Antoine R. Dumont (@ardumont) Fri, 10 Feb 2017 17:34:35 +0100 swh-loader-git (0.0.24-1~swh1) unstable-swh; urgency=medium * v0.0.24 * Update storage configuration reading -- Antoine R. Dumont (@ardumont) Thu, 15 Dec 2016 18:40:29 +0100 swh-loader-git (0.0.23-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.23 * Make the save_data mechanism generic -- Nicolas Dandrimont Fri, 02 Dec 2016 15:34:05 +0100 swh-loader-git (0.0.22-1~swh1) unstable-swh; urgency=medium * v0.0.22 * Improve reader to permit to use it as analyzer tool -- Antoine R. Dumont (@ardumont) Fri, 04 Nov 2016 10:37:24 +0100 swh-loader-git (0.0.21-1~swh1) unstable-swh; urgency=medium * v0.0.21 * Improve the reader git to load all contents from a pack. * Improve to avoid unnecessary readings from db -- Antoine R. Dumont (@ardumont) Wed, 26 Oct 2016 17:06:12 +0200 swh-loader-git (0.0.20-1~swh1) unstable-swh; urgency=medium * v0.0.20 * Add new reader git task -- Antoine R. Dumont (@ardumont) Tue, 25 Oct 2016 18:40:17 +0200 swh-loader-git (0.0.19-1~swh1) unstable-swh; urgency=medium * v0.0.19 * Update git loaders to register origin_visit's state -- Antoine R. Dumont (@ardumont) Tue, 23 Aug 2016 16:34:15 +0200 swh-loader-git (0.0.18-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.18 * Properly handle skipped contents -- Nicolas Dandrimont Fri, 19 Aug 2016 18:12:44 +0200 swh-loader-git (0.0.16-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.16 * Add exist_ok to packfile cache directory creation -- Nicolas Dandrimont Mon, 01 Aug 2016 15:53:07 +0200 swh-loader-git (0.0.15-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.15 * Absence of remote refs doesn't throw an error in updater -- Nicolas Dandrimont Wed, 15 Jun 2016 01:20:37 +0200 swh-loader-git (0.0.14-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.14 * Add a disk loader using dulwich * Rework the loader logic to use a single pattern for both loaders * Allow caching of packfiles for the remote loader -- Nicolas Dandrimont Tue, 14 Jun 2016 18:10:21 +0200 swh-loader-git (0.0.13-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.13 * Update for latest schema revision -- Nicolas Dandrimont Fri, 08 Apr 2016 16:46:41 +0200 swh-loader-git (0.0.12-1~swh1) unstable-swh; urgency=medium * Release swh-loader-git v0.0.12 * Update to use new swh.storage api for object listing * Add a size limit to packfiles * Return a proper eventfulness for empty repositories * Do not crawl the pack file if unnecessary -- Nicolas Dandrimont Thu, 25 Feb 2016 18:21:34 +0100 swh-loader-git (0.0.11-1~swh1) unstable-swh; urgency=medium * Release swh.loader.git v0.0.11 * Implement git updater -- Nicolas Dandrimont Fri, 19 Feb 2016 19:13:22 +0100 swh-loader-git (0.0.10-1~swh1) unstable-swh; urgency=medium * Prepare swh.loader.git release v0.0.10 * Update for swh.model * Use new swh.storage -- Nicolas Dandrimont Mon, 07 Dec 2015 18:59:46 +0100 swh-loader-git (0.0.9-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.loader.git v0.0.9 * Close fetch_history on failure too -- Nicolas Dandrimont Wed, 04 Nov 2015 10:54:37 +0100 swh-loader-git (0.0.8-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.loader.git v0.0.8 * New database schema (v028) * Populate fetch_history (T121) -- Nicolas Dandrimont Tue, 27 Oct 2015 18:11:26 +0100 swh-loader-git (0.0.7-1~swh1) unstable-swh; urgency=medium * Prepare swh.loader.git v0.0.7 deployment -- Nicolas Dandrimont Mon, 19 Oct 2015 12:37:09 +0200 swh-loader-git (0.0.6-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.loader.git v0.0.6 -- Nicolas Dandrimont Fri, 09 Oct 2015 17:50:35 +0200 swh-loader-git (0.0.5-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.loader.git v0.0.5 -- Nicolas Dandrimont Tue, 06 Oct 2015 17:42:11 +0200 swh-loader-git (0.0.4-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.loader.git v0.0.4 -- Nicolas Dandrimont Fri, 02 Oct 2015 14:54:04 +0200 swh-loader-git (0.0.3-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.loader.git v0.0.3 -- Nicolas Dandrimont Thu, 01 Oct 2015 11:36:28 +0200 swh-loader-git (0.0.2-1~swh1) unstable-swh; urgency=medium * Prepare deploying swh.loader.git v0.0.2 -- Nicolas Dandrimont Tue, 29 Sep 2015 17:22:09 +0200 swh-loader-git (0.0.1-1~swh1) unstable-swh; urgency=medium * Initial release * Tagging swh.loader.git v0.0.1 -- Nicolas Dandrimont Fri, 25 Sep 2015 16:04:00 +0200 diff --git a/swh.loader.git.egg-info/PKG-INFO b/swh.loader.git.egg-info/PKG-INFO index f6876cb..f0ba53c 100644 --- a/swh.loader.git.egg-info/PKG-INFO +++ b/swh.loader.git.egg-info/PKG-INFO @@ -1,106 +1,106 @@ Metadata-Version: 2.1 Name: swh.loader.git -Version: 1.1.2 +Version: 1.1.3 Summary: Software Heritage git loader Home-page: https://forge.softwareheritage.org/diffusion/DLDG/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-loader-git Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-loader-git/ Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing License-File: LICENSE License-File: AUTHORS swh-loader-git ============== The Software Heritage Git Loader is a tool and a library to walk a local Git repository and inject into the SWH dataset all contained files that weren't known before. The main entry points are - :class:`swh.loader.git.loader.GitLoader` for the main loader which ingests a remote git repository's contents. - :class:`swh.loader.git.from_disk.GitLoaderFromDisk` which ingests a local git clone repository. - :class:`swh.loader.git.loader.GitLoaderFromArchive` which ingests a git repository wrapped in an archive. License ------- This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. See top-level LICENSE file for the full text of the GNU General Public License along with this program. Dependencies ------------ ### Runtime - python3 - python3-dulwich - python3-retrying - python3-swh.core - python3-swh.model - python3-swh.storage - python3-swh.scheduler ### Test - python3-nose Requirements ------------ - implementation language, Python3 - coding guidelines: conform to PEP8 - Git access: via dulwich CLI Run ---------- You can run the loader from a remote origin (*loader*) or from an origin on disk (*from_disk*) directly by calling: ``` swh loader -C run git ``` or "git_disk". ## Configuration sample /tmp/git.yml: ``` storage: cls: remote args: url: http://localhost:5002/ ``` diff --git a/swh/loader/git/converters.py b/swh/loader/git/converters.py index 3f733a8..a95d3bb 100644 --- a/swh/loader/git/converters.py +++ b/swh/loader/git/converters.py @@ -1,216 +1,216 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Convert dulwich objects to dictionaries suitable for swh.storage""" from typing import Any, Dict, Optional, cast from dulwich.objects import Blob, Commit, ShaFile, Tag, Tree from swh.model.hashutil import DEFAULT_ALGORITHMS, MultiHash, hash_to_bytes from swh.model.model import ( BaseContent, Content, Directory, DirectoryEntry, HashableObject, ObjectType, Person, Release, Revision, RevisionType, SkippedContent, TargetType, Timestamp, TimestampWithTimezone, ) class HashMismatch(Exception): pass def check_id(obj: HashableObject) -> None: real_id = obj.compute_hash() if obj.id != real_id: raise HashMismatch( f"Expected {type(obj).__name__} hash to be {obj.id.hex()}, " f"got {real_id.hex()}" ) def dulwich_blob_to_content_id(obj: ShaFile) -> Dict[str, Any]: """Convert a dulwich blob to a Software Heritage content id""" if obj.type_name != b"blob": raise ValueError("Argument is not a blob.") blob = cast(Blob, obj) size = blob.raw_length() data = blob.as_raw_string() hashes = MultiHash.from_data(data, DEFAULT_ALGORITHMS).digest() if hashes["sha1_git"] != blob.sha().digest(): raise HashMismatch( f"Expected Content hash to be {blob.sha().digest().hex()}, " f"got {hashes['sha1_git'].hex()}" ) hashes["length"] = size return hashes def dulwich_blob_to_content(obj: ShaFile, max_content_size=None) -> BaseContent: """Convert a dulwich blob to a Software Heritage content """ if obj.type_name != b"blob": raise ValueError("Argument is not a blob.") blob = cast(Blob, obj) hashes = dulwich_blob_to_content_id(blob) if max_content_size is not None and hashes["length"] >= max_content_size: return SkippedContent(status="absent", reason="Content too large", **hashes,) else: return Content(data=blob.as_raw_string(), status="visible", **hashes,) def dulwich_tree_to_directory(obj: ShaFile) -> Directory: """Format a tree as a directory""" if obj.type_name != b"tree": raise ValueError("Argument is not a tree.") tree = cast(Tree, obj) entries = [] entry_mode_map = { 0o040000: "dir", 0o160000: "rev", 0o100644: "file", 0o100755: "file", 0o120000: "file", } for entry in tree.iteritems(): entries.append( DirectoryEntry( type=entry_mode_map.get(entry.mode, "file"), perms=entry.mode, name=entry.path, target=hash_to_bytes(entry.sha.decode("ascii")), ) ) dir_ = Directory(id=tree.sha().digest(), entries=tuple(entries),) check_id(dir_) return dir_ def parse_author(name_email: bytes) -> Person: """Parse an author line""" return Person.from_fullname(name_email) def dulwich_tsinfo_to_timestamp( timestamp, timezone, timezone_neg_utc ) -> TimestampWithTimezone: """Convert the dulwich timestamp information to a structure compatible with Software Heritage""" return TimestampWithTimezone( timestamp=Timestamp(seconds=int(timestamp), microseconds=0,), offset=timezone // 60, negative_utc=timezone_neg_utc if timezone == 0 else False, ) def dulwich_commit_to_revision(obj: ShaFile) -> Revision: if obj.type_name != b"commit": raise ValueError("Argument is not a commit.") commit = cast(Commit, obj) extra_headers = [] if commit.encoding is not None: extra_headers.append((b"encoding", commit.encoding)) if commit.mergetag: for mergetag in commit.mergetag: raw_string = mergetag.as_raw_string() assert raw_string.endswith(b"\n") extra_headers.append((b"mergetag", raw_string[:-1])) if commit.extra: extra_headers.extend((k, v) for k, v in commit.extra) if commit.gpgsig: extra_headers.append((b"gpgsig", commit.gpgsig)) rev = Revision( id=commit.sha().digest(), author=parse_author(commit.author), date=dulwich_tsinfo_to_timestamp( commit.author_time, commit.author_timezone, commit._author_timezone_neg_utc, ), committer=parse_author(commit.committer), committer_date=dulwich_tsinfo_to_timestamp( commit.commit_time, commit.commit_timezone, commit._commit_timezone_neg_utc, ), type=RevisionType.GIT, directory=bytes.fromhex(commit.tree.decode()), message=commit.message, metadata=None, extra_headers=tuple(extra_headers), synthetic=False, parents=tuple(bytes.fromhex(p.decode()) for p in commit.parents), ) check_id(rev) return rev DULWICH_TARGET_TYPES = { b"blob": TargetType.CONTENT, b"tree": TargetType.DIRECTORY, b"commit": TargetType.REVISION, b"tag": TargetType.RELEASE, } DULWICH_OBJECT_TYPES = { b"blob": ObjectType.CONTENT, b"tree": ObjectType.DIRECTORY, b"commit": ObjectType.REVISION, b"tag": ObjectType.RELEASE, } def dulwich_tag_to_release(obj: ShaFile) -> Release: if obj.type_name != b"tag": raise ValueError("Argument is not a tag.") tag = cast(Tag, obj) target_type, target = tag.object if tag.tagger: author: Optional[Person] = parse_author(tag.tagger) - if not tag.tag_time: + if tag.tag_time is None: date = None else: date = dulwich_tsinfo_to_timestamp( tag.tag_time, tag.tag_timezone, tag._tag_timezone_neg_utc, ) else: author = date = None message = tag.message if tag.signature: message += tag.signature rel = Release( id=tag.sha().digest(), author=author, date=date, name=tag.name, target=bytes.fromhex(target.decode()), target_type=DULWICH_OBJECT_TYPES[target_type.type_name], message=message, metadata=None, synthetic=False, ) check_id(rel) return rel diff --git a/swh/loader/git/tests/test_converters.py b/swh/loader/git/tests/test_converters.py index 71967d8..697a2e5 100644 --- a/swh/loader/git/tests/test_converters.py +++ b/swh/loader/git/tests/test_converters.py @@ -1,562 +1,607 @@ # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import copy +import datetime import os import shutil import subprocess import tempfile import dulwich.repo import pytest import swh.loader.git.converters as converters from swh.model.hashutil import bytehex_to_hash, hash_to_bytehex, hash_to_bytes from swh.model.model import ( Content, ObjectType, Person, Release, Revision, RevisionType, Timestamp, TimestampWithTimezone, ) TEST_DATA = os.path.join(os.path.dirname(__file__), "data") GPGSIG = ( b"-----BEGIN PGP SIGNATURE-----\n" b"\n" b"iQJLBAABCAA1FiEEAOWDevQbOk/9ITMF6ImSleOlnUcFAl8EnS4XHGRhdmlkLmRv\n" b"dWFyZEBzZGZhMy5vcmcACgkQ6ImSleOlnUdrqQ/8C5RO4NZ5Qr/dwAy2cPA7ktkY\n" b"1oUjKtspQoPbC1X3MXVa1aWo9B3KuOMR2URw44RhMNFwjccLOhfss06E8p7CZr2H\n" b"uR3CzdDw7i52jHLCL2M2ZMaPAEbQuHjXWiUWIUXz9So8YwpTyd2XQneyOC2RDDEI\n" b"I2NVbmiMeDz33jJYPrQO0QayW+ErW+xgBF7N/qS9jFWsdV1ZNfn9NxkTH8UdGuAX\n" b"583P+0tVC2DjXc6vORVhyFzyfn1A9wHosbtWI2Mpa+zezPjoPSkcyQAJu2GyOkMC\n" b"YzSjJdQVqyovo+INkIf6PuUNdp41886BG/06xwT8fl4sVsyO51lNIfgH0DMwfTTB\n" b"ZgThYnvvO7SrXDm3QzBTXkvAiHiFFl3iNyGkCyxvgVmaTntuFT+cP+HD/pCiGaC+\n" b"jHzRwfUrmuLd/lLPyq3JXBibyjnfd3SVS+7q1NZHJ4WUmCboZ0+pfrEl65mEQ/Hz\n" b"J1qCwQ/3SsTB77ANf6lLzGSowjjrtHcBTkTbFxR4ACUhiBbosyDKpHTM7fzGFGjo\n" b"EIjohzrEnqR3bbyxJkK+nxoOByhIRdowgyeJ02I4neMyLJqcaup8NMWCddxqjaPt\n" b"YobghnjaDqEd+suL/v83hbZUAZHNO3i1OZYGMqzp1WHikDPoTwGP76baqBoXi56T\n" b"4WSpxCAJRDODHLk1HgU=\n" b"=73wF" b"\n" b"-----END PGP SIGNATURE-----" ) MERGETAG = ( b"object 9768d0b576dbaaecd80abedad6dfd0d72f1476da\n" b"type commit\n" b"tag v0.0.1\n" b"tagger David Douard 1594138133 +0200\n" b"\n" b"v0.0.1\n" b"-----BEGIN PGP SIGNATURE-----\n" b"\n" b"iQJLBAABCAA1FiEEAOWDevQbOk/9ITMF6ImSleOlnUcFAl8EnhkXHGRhdmlkLmRv\n" b"dWFyZEBzZGZhMy5vcmcACgkQ6ImSleOlnUcdzg//ZW9y2xU5JFQuUsBe/LfKrs+m\n" b"0ohVInPKXwAfpB3+gn/XtTSLe+Nnr8+QEZyVRCUz2gpGZ2tNqRjhYLIX4x5KKlaV\n" b"rfl/6Cy7zibsxxuzA1h7HylCs3IPsueQpznVHUwD9jQ5baGJSc2Lt1LufXTueHZJ\n" b"Oc0oLiP5xCZcPqeX8R/4zUUImJZ1QrPeKmQ/3F+Iq62iWp7nWDp8PtwpykSiYlNf\n" b"KrJM8omGvrlrWLtfPNUaQFClXwnwK1/HyNY2kYan6K5NtsIl2UX0LZ42GkRjJIrb\n" b"q4TFIZWZ6xndtEhHEX6B8Q5TZV6sqPgNnfGpbhj8BDoZgjD0Y43fzfDiZ0Bl2tph\n" b"tXaLg3SX/UUjFVzC1zkoQ2MR7+j8NVKauAsBINpKF4pMGsrsVRk8764pgO49iQ+S\n" b"8JVCVV76dNNm1gd7BbhFAdIAiegBtsEF69niJBoHKYLlrT8E8hDkF/gk4IkimPqf\n" b"UHtw/fPhVW3B4G2skd013NJGcnRj5oKtaM99d2Roxc3vhSRiTsoaM8BM9NDvLmJg\n" b"35rWEOnet39iJIMCHk3AYaJl8QmUhllDdr6vygaBVeVEf27m2c3NzONmIKpWqa2J\n" b"kTpF4cmzHYro34G7WuJ1bYvmLb6qWNQt9wd8RW+J1kVm5I8dkjPzLUougBpOd0YL\n" b"Bl5UTQILbV4Tv8ZlmJM=\n" b"=s1lv\n" b"-----END PGP SIGNATURE-----" ) class SWHObjectType: """Dulwich lookalike ObjectType class """ def __init__(self, type_name): self.type_name = type_name class SWHTag: """Dulwich lookalike tag class """ def __init__( self, sha, name, type_name, target, target_type, tagger, tag_time, tag_timezone, message, signature, ): self._sha = sha self.name = name self.type_name = type_name self.object = SWHObjectType(target_type), target self.tagger = tagger self.message = message self.signature = signature self.tag_time = tag_time self.tag_timezone = tag_timezone self._tag_timezone_neg_utc = False def sha(self): class hasher: def digest(): return self._sha return hasher @pytest.mark.fs class TestConverters: @classmethod def setup_class(cls): cls.repo_path = tempfile.mkdtemp() bundle = os.path.join(TEST_DATA, "git-repos", "example-submodule.bundle") git = subprocess.Popen( ["git", "clone", "--quiet", "--bare", "--mirror", bundle, cls.repo_path], cwd=TEST_DATA, ) # flush stdout of xz git.communicate() cls.repo = dulwich.repo.Repo(cls.repo_path) @classmethod def tearDownClass(cls): super().tearDownClass() shutil.rmtree(cls.repo_path) def test_blob_to_content(self): content_id = b"28c6f4023d65f74e3b59a2dea3c4277ed9ee07b0" content = converters.dulwich_blob_to_content(self.repo[content_id]) expected_content = Content( sha1_git=bytehex_to_hash(content_id), sha1=hash_to_bytes("4850a3420a2262ff061cb296fb915430fa92301c"), sha256=hash_to_bytes( "fee7c8a485a10321ad94b64135073cb5" "5f22cb9f57fa2417d2adfb09d310adef" ), blake2s256=hash_to_bytes( "5d71873f42a137f6d89286e43677721e574" "1fa05ce4cd5e3c7ea7c44d4c2d10b" ), data=( b'[submodule "example-dependency"]\n' b"\tpath = example-dependency\n" b"\turl = https://github.com/githubtraining/" b"example-dependency.git\n" ), length=124, status="visible", ) assert content == expected_content def test_corrupt_blob(self, mocker): # has a signature sha1 = hash_to_bytes("28c6f4023d65f74e3b59a2dea3c4277ed9ee07b0") blob = copy.deepcopy(self.repo[hash_to_bytehex(sha1)]) class hasher: def digest(): return sha1 blob._sha = hasher converters.dulwich_blob_to_content(blob) converters.dulwich_blob_to_content_id(blob) sha1 = hash_to_bytes("1234" * 10) with pytest.raises(converters.HashMismatch): converters.dulwich_blob_to_content(blob) with pytest.raises(converters.HashMismatch): converters.dulwich_blob_to_content_id(blob) def test_convertion_wrong_input(self): class Something: type_name = b"something-not-the-right-type" m = { "blob": converters.dulwich_blob_to_content, "tree": converters.dulwich_tree_to_directory, "commit": converters.dulwich_tree_to_directory, "tag": converters.dulwich_tag_to_release, } for _callable in m.values(): with pytest.raises(ValueError): _callable(Something()) def test_corrupt_tree(self): # has a signature sha1 = b"f0695c2e2fa7ce9d574023c3413761a473e500ca" tree = copy.deepcopy(self.repo[sha1]) converters.dulwich_tree_to_directory(tree) del tree._entries[next(iter(tree._entries))] with pytest.raises(converters.HashMismatch): converters.dulwich_tree_to_directory(tree) def test_commit_to_revision(self): sha1 = b"9768d0b576dbaaecd80abedad6dfd0d72f1476da" revision = converters.dulwich_commit_to_revision(self.repo[sha1]) expected_revision = Revision( id=hash_to_bytes("9768d0b576dbaaecd80abedad6dfd0d72f1476da"), directory=b"\xf0i\\./\xa7\xce\x9dW@#\xc3A7a\xa4s\xe5\x00\xca", type=RevisionType.GIT, committer=Person( name=b"Stefano Zacchiroli", fullname=b"Stefano Zacchiroli ", email=b"zack@upsilon.cc", ), author=Person( name=b"Stefano Zacchiroli", fullname=b"Stefano Zacchiroli ", email=b"zack@upsilon.cc", ), committer_date=TimestampWithTimezone( timestamp=Timestamp(seconds=1443083765, microseconds=0,), negative_utc=False, offset=120, ), message=b"add submodule dependency\n", metadata=None, extra_headers=(), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1443083765, microseconds=0,), negative_utc=False, offset=120, ), parents=(b"\xc3\xc5\x88q23`\x9f[\xbb\xb2\xd9\xe7\xf3\xfbJf\x0f?r",), synthetic=False, ) assert revision == expected_revision def test_commit_to_revision_with_extra_headers(self): sha1 = b"322f5bc915e50fc25e85226b5a182bded0e98e4b" revision = converters.dulwich_commit_to_revision(self.repo[sha1]) expected_revision = Revision( id=hash_to_bytes(sha1.decode()), directory=bytes.fromhex("f8ec06e4ed7b9fff4918a0241a48023143f30000"), type=RevisionType.GIT, committer=Person( name=b"David Douard", fullname=b"David Douard ", email=b"david.douard@sdfa3.org", ), author=Person( name=b"David Douard", fullname=b"David Douard ", email=b"david.douard@sdfa3.org", ), committer_date=TimestampWithTimezone( timestamp=Timestamp(seconds=1594137902, microseconds=0,), negative_utc=False, offset=120, ), message=b"Am\xe9lioration du fichier READM\xa4\n", metadata=None, extra_headers=((b"encoding", b"ISO-8859-15"), (b"gpgsig", GPGSIG)), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1594136900, microseconds=0,), negative_utc=False, offset=120, ), parents=(bytes.fromhex("c730509025c6e81947102b2d77bc4dc1cade9489"),), synthetic=False, ) assert revision == expected_revision @pytest.mark.parametrize( "attribute", ["_message", "_encoding", "_author", "_gpgsig"] ) def test_corrupt_commit(self, attribute): # has a signature sha1 = b"322f5bc915e50fc25e85226b5a182bded0e98e4b" commit = copy.deepcopy(self.repo[sha1]) converters.dulwich_commit_to_revision(commit) setattr(commit, attribute, b"abcde") with pytest.raises(converters.HashMismatch): converters.dulwich_commit_to_revision(commit) if attribute == "_gpgsig": setattr(commit, attribute, None) with pytest.raises(converters.HashMismatch): converters.dulwich_commit_to_revision(commit) def test_commit_to_revision_with_extra_headers_mergetag(self): sha1 = b"3ab3da4bf0f81407be16969df09cd1c8af9ac703" revision = converters.dulwich_commit_to_revision(self.repo[sha1]) expected_revision = Revision( id=hash_to_bytes(sha1.decode()), directory=bytes.fromhex("faa4b64a841ca3e3f07d6501caebda2e3e8e544e"), type=RevisionType.GIT, committer=Person( name=b"David Douard", fullname=b"David Douard ", email=b"david.douard@sdfa3.org", ), author=Person( name=b"David Douard", fullname=b"David Douard ", email=b"david.douard@sdfa3.org", ), committer_date=TimestampWithTimezone( timestamp=Timestamp(seconds=1594138183, microseconds=0,), negative_utc=False, offset=120, ), message=b"Merge tag 'v0.0.1' into readme\n\nv0.0.1\n", metadata=None, extra_headers=((b"encoding", b"ISO-8859-15"), (b"mergetag", MERGETAG)), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1594138183, microseconds=0,), negative_utc=False, offset=120, ), parents=( bytes.fromhex("322f5bc915e50fc25e85226b5a182bded0e98e4b"), bytes.fromhex("9768d0b576dbaaecd80abedad6dfd0d72f1476da"), ), synthetic=False, ) assert revision == expected_revision def test_author_line_to_author(self): # edge case out of the way with pytest.raises(TypeError): converters.parse_author(None) tests = { b"a ": Person( name=b"a", email=b"b@c.com", fullname=b"a ", ), b"": Person( name=None, email=b"foo@bar.com", fullname=b"", ), b"malformed ": Person( name=b"trailing", email=b"sp@c.e", fullname=b"trailing ", ), b"no": Person(name=b"no", email=b"sp@c.e", fullname=b"no",), b" <>": Person(name=None, email=None, fullname=b" <>",), b"something": Person(name=b"something", email=None, fullname=b"something"), } for author in sorted(tests): parsed_author = tests[author] assert parsed_author == converters.parse_author(author) def test_dulwich_tag_to_release_no_author_no_date(self): sha = hash_to_bytes("f6e367357b446bd1315276de5e88ba3d0d99e136") target = b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce" message = b"some release message" tag = SWHTag( sha=sha, name=b"blah", type_name=b"tag", target=target, target_type=b"commit", message=message, signature=None, tagger=None, tag_time=None, tag_timezone=None, ) # when actual_release = converters.dulwich_tag_to_release(tag) # then expected_release = Release( author=None, date=None, id=sha, message=message, metadata=None, name=b"blah", synthetic=False, target=hash_to_bytes(target.decode()), target_type=ObjectType.REVISION, ) assert actual_release == expected_release def test_dulwich_tag_to_release_author_and_date(self): sha = hash_to_bytes("fc1e6a4f1e37e93e28e78560e73efd0b12f616ef") tagger = b"hey dude " target = b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce" message = b"some release message" - import datetime - date = datetime.datetime(2007, 12, 5, tzinfo=datetime.timezone.utc).timestamp() tag = SWHTag( sha=sha, name=b"blah", type_name=b"tag", target=target, target_type=b"commit", message=message, signature=None, tagger=tagger, tag_time=date, tag_timezone=0, ) # when actual_release = converters.dulwich_tag_to_release(tag) # then expected_release = Release( author=Person( email=b"hello@mail.org", fullname=b"hey dude ", name=b"hey dude", ), date=TimestampWithTimezone( negative_utc=False, offset=0, timestamp=Timestamp(seconds=1196812800, microseconds=0,), ), id=sha, message=message, metadata=None, name=b"blah", synthetic=False, target=hash_to_bytes(target.decode()), target_type=ObjectType.REVISION, ) assert actual_release == expected_release def test_dulwich_tag_to_release_author_no_date(self): # to reproduce bug T815 (fixed) sha = hash_to_bytes("41076e970975122dc6b2a878aa9797960bc4781d") tagger = b"hey dude " target = b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce" message = b"some release message" tag = SWHTag( sha=sha, name=b"blah", type_name=b"tag", target=target, target_type=b"commit", message=message, signature=None, tagger=tagger, tag_time=None, tag_timezone=None, ) # when actual_release = converters.dulwich_tag_to_release(tag) # then expected_release = Release( author=Person( email=b"hello@mail.org", fullname=b"hey dude ", name=b"hey dude", ), date=None, id=sha, message=message, metadata=None, name=b"blah", synthetic=False, target=hash_to_bytes(target.decode()), target_type=ObjectType.REVISION, ) assert actual_release == expected_release + def test_dulwich_tag_to_release_author_zero_date(self): + # to reproduce bug T815 (fixed) + sha = hash_to_bytes("6cc1deff5cdcd853428bb63b937f43dd2566c36f") + tagger = b"hey dude " + target = b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce" + message = b"some release message" + date = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc).timestamp() + tag = SWHTag( + sha=sha, + name=b"blah", + type_name=b"tag", + target=target, + target_type=b"commit", + message=message, + signature=None, + tagger=tagger, + tag_time=date, + tag_timezone=0, + ) + + # when + actual_release = converters.dulwich_tag_to_release(tag) + + # then + expected_release = Release( + author=Person( + email=b"hello@mail.org", + fullname=b"hey dude ", + name=b"hey dude", + ), + date=TimestampWithTimezone( + negative_utc=False, + offset=0, + timestamp=Timestamp(seconds=0, microseconds=0,), + ), + id=sha, + message=message, + metadata=None, + name=b"blah", + synthetic=False, + target=hash_to_bytes(target.decode()), + target_type=ObjectType.REVISION, + ) + + assert actual_release == expected_release + def test_dulwich_tag_to_release_signature(self): target = b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce" message = b"some release message" sha = hash_to_bytes("46fff489610ed733d2cc904e363070dadee05c71") tag = SWHTag( sha=sha, name=b"blah", type_name=b"tag", target=target, target_type=b"commit", message=message, signature=GPGSIG, tagger=None, tag_time=None, tag_timezone=None, ) # when actual_release = converters.dulwich_tag_to_release(tag) # then expected_release = Release( author=None, date=None, id=sha, message=message + GPGSIG, metadata=None, name=b"blah", synthetic=False, target=hash_to_bytes(target.decode()), target_type=ObjectType.REVISION, ) assert actual_release == expected_release @pytest.mark.parametrize("attribute", ["name", "message", "signature"]) def test_corrupt_tag(self, attribute): # has a signature sha = hash_to_bytes("46fff489610ed733d2cc904e363070dadee05c71") target = b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce" message = b"some release message" tag = SWHTag( sha=sha, name=b"blah", type_name=b"tag", target=target, target_type=b"commit", message=message, signature=GPGSIG, tagger=None, tag_time=None, tag_timezone=None, ) converters.dulwich_tag_to_release(tag) setattr(tag, attribute, b"abcde") with pytest.raises(converters.HashMismatch): converters.dulwich_tag_to_release(tag) if attribute == "signature": setattr(tag, attribute, None) with pytest.raises(converters.HashMismatch): converters.dulwich_tag_to_release(tag)