diff --git a/PKG-INFO b/PKG-INFO index 376f27c..045f03f 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,39 +1,39 @@ Metadata-Version: 2.1 Name: swh.core -Version: 2.15.0 +Version: 2.16.0 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-core Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-core/ Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/x-rst Provides-Extra: testing-core Provides-Extra: logging Provides-Extra: db Provides-Extra: http Provides-Extra: github Provides-Extra: testing License-File: LICENSE License-File: AUTHORS Software Heritage - Core foundations ==================================== Low-level utilities and helpers used by almost all other modules in the stack. core library for swh's modules: - config parser - serialization - logging mechanism - database connection - http-based RPC client/server diff --git a/debian/changelog b/debian/changelog index 211679b..3f47d39 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,1465 +1,1467 @@ -swh-core (2.15.0-1~swh1~bpo10+1) buster-swh; urgency=medium +swh-core (2.16.0-1~swh1) unstable-swh; urgency=medium - * Rebuild for buster-swh + * New upstream release 2.16.0 - (tagged by Antoine Lambert + on 2022-10-12 11:34:18 +0200) + * Upstream changes: - version 2.16.0 - -- Software Heritage autobuilder (on jenkins-debian1) Mon, 03 Oct 2022 15:09:23 +0000 + -- Software Heritage autobuilder (on jenkins-debian1) Wed, 12 Oct 2022 09:37:28 +0000 swh-core (2.15.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.15.0 - (tagged by Antoine R. Dumont (@ardumont) on 2022-10-03 17:04:49 +0200) * Upstream changes: - v2.15.0 - Make mimetype to archive format dictionary public - api/asynchronous: Do not log/report client exception -- Software Heritage autobuilder (on jenkins-debian1) Mon, 03 Oct 2022 15:07:52 +0000 swh-core (2.14.1-1~swh1) unstable-swh; urgency=medium * New upstream release 2.14.1 - (tagged by Antoine Lambert on 2022-09-12 11:46:16 +0200) * Upstream changes: - version 2.14.1 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 12 Sep 2022 09:51:15 +0000 swh-core (2.14.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.14.0 - (tagged by Valentin Lorentz on 2022-08-16 13:49:11 +0200) * Upstream changes: - v2.14.0 - * RPC server: Do not log exceptions with 4xx HTTP status codes - * Remove support for deprecated exception format - * Make the RPC client raise a specific exception class on 503 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 16 Aug 2022 11:52:17 +0000 swh-core (2.13.1-1~swh1) unstable-swh; urgency=medium * New upstream release 2.13.1 - (tagged by Valentin Lorentz on 2022-08-04 10:42:39 +0200) * Upstream changes: - v2.13.1 - * Remove use of app.test_client() as a ctx-manager, to fix support - for Flask 2.2.0 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 04 Aug 2022 08:45:44 +0000 swh-core (2.13-1~swh1) unstable-swh; urgency=medium * New upstream release 2.13 - (tagged by David Douard on 2022-07-05 14:22:04 +0200) * Upstream changes: - v2.13 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 05 Jul 2022 12:38:24 +0000 swh-core (2.12-1~swh2) unstable-swh; urgency=medium * Bump new release with correct deps. -- Antoine R. Dumont (@ardumont) Mon, 20 Jun 2022 10:13:14 +0200 swh-core (2.12-1~swh1) unstable-swh; urgency=medium * New upstream release 2.12 - (tagged by Nicolas Dandrimont on 2022-06-16 13:38:52 +0200) * Upstream changes: - Release swh.core v2.12 - Add support for zstandard compressed tarballs -- Software Heritage autobuilder (on jenkins-debian1) Thu, 16 Jun 2022 11:42:04 +0000 swh-core (2.11-1~swh1) unstable-swh; urgency=medium * New upstream release 2.11 - (tagged by Antoine R. Dumont (@ardumont) on 2022-06-10 10:20:37 +0200) * Upstream changes: - v2.11 - tarball: Use standard Python module zipfile to extract jar archive - Add missing coverage on `swh db version` cli - db: Grant read access to guest user on all schema tables -- Software Heritage autobuilder (on jenkins-debian1) Fri, 10 Jun 2022 08:24:24 +0000 swh-core (2.10-1~swh1) unstable-swh; urgency=medium * New upstream release 2.10 - (tagged by Antoine R. Dumont (@ardumont) on 2022-06-02 16:03:41 +0200) * Upstream changes: - v2.10 - github/utils: Deal with exotic urls to canonicalize - deprecate the db/pytest_plugin.py module - move initialize_database_for_module in db_utils - mark postgresql_fact fixture factory function as deprecated - tests: use stock pytest_postgresql factory function - docs/db: Update datastore requirement -- Software Heritage autobuilder (on jenkins-debian1) Thu, 02 Jun 2022 14:39:11 +0000 swh-core (2.9.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.9.0 - (tagged by Antoine R. Dumont (@ardumont) on 2022-05-30 17:36:23 +0200) * Upstream changes: - v2.9.0 - Allow module to specify another config key than their module name - cli.db: Reword ignore sentence -- Software Heritage autobuilder (on jenkins-debian1) Mon, 30 May 2022 15:41:42 +0000 swh-core (2.8.1-1~swh1) unstable-swh; urgency=medium * New upstream release 2.8.1 - (tagged by Antoine R. Dumont (@ardumont) on 2022-05-30 17:06:11 +0200) * Upstream changes: - v2.8.1 - cli.db: Use attribute current_version instead of undeclared getter - cli.db: Fix help message typo -- Software Heritage autobuilder (on jenkins-debian1) Mon, 30 May 2022 15:10:17 +0000 swh-core (2.8.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.8.0 - (tagged by Antoine R. Dumont (@ardumont) on 2022-05-20 18:17:20 +0200) * Upstream changes: - v2.8.0 - Deal with git protocol url to canonicalize to https -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 May 2022 16:21:13 +0000 swh-core (2.7.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.7.0 - (tagged by Antoine R. Dumont (@ardumont) on 2022-05-20 15:26:30 +0200) * Upstream changes: - v2.7.0 - Use GitHubSession to make canonical computation deal with rate limit -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 May 2022 13:30:15 +0000 swh-core (2.6.0-1~swh2) unstable-swh; urgency=medium * Bump new release -- Antoine R. Dumont (@ardumont) Fri, 20 May 2022 10:26:03 +0200 swh-core (2.6.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.6.0 - (tagged by Antoine R. Dumont (@ardumont) on 2022-05-20 08:54:46 +0200) * Upstream changes: - v2.6.0 - Extract reusable github tests fixtures into its own pytest_plugin - test_db/test_db_copy_to: Fix hypothesis FailedHealthCheck error - Refactor swh.lister.github.utils to swh.core.github.utils - Add utility function to retrieve canonical github urls - Make 'python -m swh' work as cli entry point - db_utils: Make connect_to_conninfo use through contextmanager - Upgrade mypy to 0.942 to fix support of types-psycopg2 >= 2.9.12 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 May 2022 06:58:23 +0000 swh-core (2.5.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.5.0 - (tagged by Valentin Lorentz on 2022-04-25 13:51:26 +0200) * Upstream changes: - v2.5.0 - * Make db_transaction's client_options configurable at run time - * sentry: always override init settings with the environment variables - * Add support for disabling logging-based events in sentry - * RPC server: explicitly handle sentry exception capture (instead of gunicorn) - * statsd: add an error_type tag to @timed error counters - * cli: Ensure tests don't mess with the global logging setup - * mypy/pre-commit/pytest maintenance -- Software Heritage autobuilder (on jenkins-debian1) Mon, 25 Apr 2022 11:56:02 +0000 swh-core (2.4.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.4.0 - (tagged by Valentin Lorentz on 2022-03-29 14:39:15 +0200) * Upstream changes: - v2.4.0 - * Fix support of Werkzeug 2.1.0 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 29 Mar 2022 12:42:28 +0000 swh-core (2.3.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.3.0 - (tagged by David Douard on 2022-03-14 17:19:37 +0100) * Upstream changes: - v2.3.0 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 14 Mar 2022 16:23:52 +0000 swh-core (2.2.2-1~swh1) unstable-swh; urgency=medium * New upstream release 2.2.2 - (tagged by David Douard on 2022-03-09 17:44:25 +0100) * Upstream changes: - v2.2.2 - small fixes in the `swh db upgrade` command -- Software Heritage autobuilder (on jenkins-debian1) Wed, 09 Mar 2022 16:48:21 +0000 swh-core (2.2.1-1~swh1) unstable-swh; urgency=medium * New upstream release 2.2.1 - (tagged by Nicolas Dandrimont on 2022-03-03 20:20:11 +0100) * Upstream changes: - Release swh.core v2.2.1 - quiesce deprecation warnings for autogenerated RPC clients - fix typing for a few methods in db_utils -- Software Heritage autobuilder (on jenkins-debian1) Thu, 03 Mar 2022 19:24:59 +0000 swh-core (2.2.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.2.0 - (tagged by Valentin Lorentz on 2022-03-02 11:45:18 +0100) * Upstream changes: - v2.2.0 - * utils: Add a new 'iter_chunks' function -- Software Heritage autobuilder (on jenkins-debian1) Wed, 02 Mar 2022 10:47:47 +0000 swh-core (2.1.1-1~swh1) unstable-swh; urgency=medium * New upstream release 2.1.1 - (tagged by Valentin Lorentz on 2022-03-01 17:00:36 +0100) * Upstream changes: - v2.1.1 - * Fix wrong version numbers in deprecation message. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 01 Mar 2022 16:03:05 +0000 swh-core (2.1.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.1.0 - (tagged by Valentin Lorentz on 2022-03-01 16:58:02 +0100) * Upstream changes: - v2.1.0 - * RPCClient: Make methods {get,post}{,_stream} protected - * RPCServerApp: Add hooks to add behaviors to generated methods -- Software Heritage autobuilder (on jenkins-debian1) Tue, 01 Mar 2022 16:00:51 +0000 swh-core (2.0.0-2~swh1) unstable-swh; urgency=medium * Bump dependency constranints for pytest and pytest-postgresql. -- David Douard Wed, 23 Feb 2022 10:56:37 +0100 swh-core (2.0.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.0.0 - (tagged by David Douard on 2022-02-17 15:26:59 +0100) * Upstream changes: - v2.0.0 - add support for generic db version handling for postgresql backends, - add support for generic db upgrade (for postgresql backends), - upgrade pytest- postgresql based tests scaffolding to use the - template-based db creation (instead of the truncate-based db reset). -- Software Heritage autobuilder (on jenkins-debian1) Wed, 23 Feb 2022 09:34:41 +0000 swh-core (1.1.1-1~swh1) unstable-swh; urgency=medium * New upstream release 1.1.1 - (tagged by Valentin Lorentz on 2022-02-04 12:52:06 +0100) * Upstream changes: - v1.1.1 - * Require pytest to be <7.0.0 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 04 Feb 2022 11:54:45 +0000 swh-core (1.1.0-1~swh1) unstable-swh; urgency=medium * New upstream release 1.1.0 - (tagged by David Douard on 2022-01-20 15:39:02 +0100) * Upstream changes: - v1.1.0 - add a Statsd.status_gauge() context manager - add support for env var substitution in STATS_TAGS - pin mypy version and clean reauirements a bit -- Software Heritage autobuilder (on jenkins-debian1) Thu, 20 Jan 2022 14:44:14 +0000 swh-core (1.0.0-2~swh1) unstable-swh; urgency=medium * Add missing B-D on python3-blinker for sentry-sdk -- Nicolas Dandrimont Thu, 02 Dec 2021 12:34:50 +0100 swh-core (1.0.0-1~swh1) unstable-swh; urgency=medium * New upstream release 1.0.0 - (tagged by David Douard on 2021-12-02 10:51:10 +0100) * Upstream changes: - v1.0.0 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 02 Dec 2021 10:00:19 +0000 swh-core (0.15.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.15.1 - (tagged by Valentin Lorentz on 2021-11-08 14:10:08 +0100) * Upstream changes: - v0.15.1 - * Require pytest-postgresql < 4.0 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 08 Nov 2021 13:12:36 +0000 swh-core (0.15.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.15.0 - (tagged by Antoine R. Dumont (@ardumont) on 2021-09-20 11:45:26 +0200) * Upstream changes: - v0.15.0 - tarball: Fallback to guess archive format from mimetype when no format detected -- Software Heritage autobuilder (on jenkins-debian1) Mon, 20 Sep 2021 09:48:28 +0000 swh-core (0.14.6-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.6 - (tagged by Antoine Lambert on 2021-09-16 10:48:35 +0200) * Upstream changes: - version 0.14.6 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 16 Sep 2021 08:51:58 +0000 swh-core (0.14.5-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.5 - (tagged by Valentin Lorentz on 2021-08-30 10:31:47 +0200) * Upstream changes: - v0.14.5 - * tarball: Add support for .tbz, .tbz2, and .jar -- Software Heritage autobuilder (on jenkins-debian1) Mon, 30 Aug 2021 08:34:03 +0000 swh-core (0.14.4-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.4 - (tagged by Valentin Lorentz on 2021-07-30 16:44:26 +0200) * Upstream changes: - v0.14.4 - * add stream_results_optional -- Software Heritage autobuilder (on jenkins-debian1) Fri, 30 Jul 2021 14:47:25 +0000 swh-core (0.14.3-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.3 - (tagged by Antoine Lambert on 2021-06-11 15:41:38 +0200) * Upstream changes: - version 0.14.3 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 11 Jun 2021 13:47:13 +0000 swh-core (0.14.2-2~swh1) unstable-swh; urgency=medium * Rebuild v0.14.2 after missing unzip dependency -- Antoine Lambert Thu, 10 Jun 2021 16:24:09 +0200 swh-core (0.14.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.2 - (tagged by Antoine Lambert on 2021-06-10 16:09:06 +0200) * Upstream changes: - version 0.14.2 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 10 Jun 2021 14:13:25 +0000 swh-core (0.14.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.1 - (tagged by Valentin Lorentz on 2021-05-06 15:33:56 +0200) * Upstream changes: - v0.14.1 - * Fix reserved name being used in pytest plugin -- Software Heritage autobuilder (on jenkins-debian1) Thu, 06 May 2021 13:38:24 +0000 swh-core (0.14.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.0 - (tagged by Valentin Lorentz on 2021-05-06 14:14:10 +0200) * Upstream changes: - v0.14.0 - Add support for pytest- postgresql 3.0.0 - For consistency, I renamed db_name to dbname everywhere, so this will - affect other SWH packages. -- Software Heritage autobuilder (on jenkins-debian1) Thu, 06 May 2021 12:17:31 +0000 swh-core (0.13.3-1~swh1) unstable-swh; urgency=medium * New upstream release 0.13.3 - (tagged by Antoine Lambert on 2021-05-06 13:54:51 +0200) * Upstream changes: - version 0.13.3 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 06 May 2021 11:59:49 +0000 swh-core (0.13.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.13.2 - (tagged by Valentin Lorentz on 2021-05-06 10:40:51 +0200) * Upstream changes: - v0.13.2 - * tarball: properly normalize perms for all extracted files - * requirements-db-pytestplugin: Don't install pytest-postgresql 3.0+ -- Software Heritage autobuilder (on jenkins-debian1) Thu, 06 May 2021 08:44:45 +0000 swh-core (0.13.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.13.1 - (tagged by Antoine Lambert on 2021-04-29 14:21:29 +0200) * Upstream changes: - version 0.13.1 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 29 Apr 2021 12:25:15 +0000 swh-core (0.13.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.13.0 - (tagged by Vincent SELLIER on 2021-04-06 19:01:37 +0200) * Upstream changes: - v0.13.0 - Support several backends on RPCServerApp -- Software Heritage autobuilder (on jenkins-debian1) Tue, 06 Apr 2021 17:07:10 +0000 swh-core (0.12.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.12.1 - (tagged by Valentin Lorentz on 2021-04-06 12:34:34 +0200) * Upstream changes: - v0.12.1 - * tests: Drop hypothesis < 6 requirement - * README.rst: Remove getting-started instructions, they are duplicates - * Improve/fix documentation of requests_mock_datadir - * Remove dependency on 'decorator' (fixes a regression in decorator 5.0.5) -- Software Heritage autobuilder (on jenkins-debian1) Tue, 06 Apr 2021 10:37:17 +0000 swh-core (0.12.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.12.0 - (tagged by David Douard on 2021-02-16 11:48:58 +0100) * Upstream changes: - v0.12.0 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 16 Feb 2021 10:52:14 +0000 swh-core (0.11.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.11.0 - (tagged by David Douard on 2020-12-08 15:35:05 +0100) * Upstream changes: - v0.11.0 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 08 Dec 2020 14:38:23 +0000 swh-core (0.10.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.10.0 - (tagged by Nicolas Dandrimont on 2020-12-02 11:46:38 +0100) * Upstream changes: - Release swh.core 0.10.0 - db.tests.db_testing: Drop unused database test utilities. - api.serializers: Add support for serializing large negative integers with - msgpack. - Do not mutate api.serializers.ENCODERS or DECODERS. - swh cli: Add support for setting up the log level of multiple loggers. -- Software Heritage autobuilder (on jenkins-debian1) Wed, 02 Dec 2020 10:50:35 +0000 swh-core (0.9.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.9.1 - (tagged by Antoine Lambert on 2020-11-23 11:24:50 +0100) * Upstream changes: - version 0.9.1 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 23 Nov 2020 10:29:15 +0000 swh-core (0.9.0-1~swh2) unstable-swh; urgency=medium * Split packages python3-swh.core and python3-swh.core.db.pytestplugin -- Antoine R. Dumont (@ardumont) Fri, 20 Nov 2020 16:37:00 +0000 swh-core (0.9.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.9.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-11-20 15:35:53 +0100) * Upstream changes: - v0.9.0 - Clarify names around the swh.core.db.pytest_plugin tests - setup: Separate pytest- postgresql dependency and declare it when needed - RPCClient: Fix reraise_exceptions regression - api/serializers: Add Exception type encoder and decoder - Makefile.local: Ensure all tests are executed when invoking make test - core.db.cli: Add coverage and ensure `swh db *` works as expected - core tests: disambiguate arg 'request' through typing -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 Nov 2020 14:37:00 +0000 swh-core (0.8.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.8.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-30 09:12:17 +0100) * Upstream changes: - v0.8.0 - cli.db: Open init-admin subcmd to initialize superuser-level scripts -- Software Heritage autobuilder (on jenkins-debian1) Fri, 30 Oct 2020 08:13:02 +0000 swh-core (0.7.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.7.1 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-22 18:51:16 +0200) * Upstream changes: - v0.7.1 - Move SWHDatabaseJanitor to db.pytest_plugin module -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Oct 2020 16:51:59 +0000 swh-core (0.7.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.7.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-22 18:31:18 +0200) * Upstream changes: - v0.7.0 - remote_api_endpoint: Allow to declare what http method to use - api.RPCServerApp: Adapt sync rpc server with async rpc server -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Oct 2020 16:32:04 +0000 swh-core (0.6.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.6.1 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-22 13:53:41 +0200) * Upstream changes: - v0.6.1 - Move pytest_plugin declaration to top-level conftest -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Oct 2020 11:54:49 +0000 swh-core (0.6.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.6.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-22 13:24:55 +0200) * Upstream changes: - v0.6.0 - asynchronous.RPCServerApp: Align implementation with api.RPCServerApp -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Oct 2020 11:25:29 +0000 swh-core (0.5.0-1~swh2) unstable-swh; urgency=medium * Bump new release with dependency updated -- Antoine R. Dumont thu, 22 Oct 2020 11:21:04 +0200 swh-core (0.5.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.5.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-22 11:11:29 +0200) * Upstream changes: - v0.5.0 - Install postgresql_fact fixture for faster postgres tests - api.tests.test_async: Simplify fixture setup - core.config: Drop no longer used SWHConfig -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Oct 2020 09:12:14 +0000 swh-core (0.4.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.4.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-02 11:44:10 +0200) * Upstream changes: - v0.4.0 - config: Deprecate SWHConfig in favor of load_from_envvar function -- Software Heritage autobuilder (on jenkins-debian1) Fri, 02 Oct 2020 09:44:53 +0000 swh-core (0.3.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.1 - (tagged by Valentin Lorentz on 2020-10-01 12:38:08 +0200) * Upstream changes: - v0.3.1 - * Add specific celery task arguments to metadata sent to systemd-journald - * SortedList: Don't inherit from UserList. -- Software Heritage autobuilder (on jenkins-debian1) Thu, 01 Oct 2020 10:40:27 +0000 swh-core (0.3.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.0 - (tagged by David Douard on 2020-09-23 16:24:40 +0200) * Upstream changes: - v0.3.0 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 23 Sep 2020 14:27:08 +0000 swh-core (0.2.3-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.3 - (tagged by Valentin Lorentz on 2020-08-17 13:55:41 +0200) * Upstream changes: - v0.2.3 - * tarball: add test for permissions. - * Move SortedList from swh-storage. -- Software Heritage autobuilder (on jenkins-debian1) Mon, 17 Aug 2020 11:57:43 +0000 swh-core (0.2.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.2 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-31 13:41:19 +0200) * Upstream changes: - v0.2.2 - api.classes: Open swh.core.api.classes.stream_results -- Software Heritage autobuilder (on jenkins-debian1) Fri, 31 Jul 2020 11:44:33 +0000 swh-core (0.2.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.1 - (tagged by Valentin Lorentz on 2020-07-30 19:16:57 +0200) * Upstream changes: - v0.2.1 - Make @remote_api_endpoint preserve typing information for mypy. -- Software Heritage autobuilder (on jenkins-debian1) Thu, 30 Jul 2020 17:20:05 +0000 swh-core (0.2.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-29 11:18:37 +0200) * Upstream changes: - v0.2.0 - core.api: Expose serializable PagedResult object for pagination api - test_serializers: Refactor using pytest - Migrate from vcversioner to setuptools- scm -- Software Heritage autobuilder (on jenkins-debian1) Wed, 29 Jul 2020 09:20:56 +0000 swh-core (0.1.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.1.2 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-08 13:41:51 +0200) * Upstream changes: - v0.1.2 - test_serializers: Move to pytest for that specific erratic assertion -- Software Heritage autobuilder (on jenkins-debian1) Wed, 08 Jul 2020 11:43:25 +0000 swh-core (0.1.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.1.1 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-08 13:03:52 +0200) * Upstream changes: - v0.1.1 - api.tests: Fix unsupported matches keyword to match - requirements-db: Move typing- extension from test to runtime deps -- Software Heritage autobuilder (on jenkins-debian1) Wed, 08 Jul 2020 11:07:55 +0000 swh-core (0.1.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.1.0 - (tagged by David Douard on 2020-07-06 14:33:28 +0200) * Upstream changes: - v0.1.0 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 06 Jul 2020 12:36:15 +0000 swh-core (0.0.95-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.95 - (tagged by Nicolas Dandrimont on 2020-04-17 17:20:35 +0200) * Upstream changes: - Release swh.core v0.0.95 - support serializing large integers in msgpack - add documentation for CLI - move formatting to black -- Software Heritage autobuilder (on jenkins-debian1) Fri, 17 Apr 2020 16:05:01 +0000 swh-core (0.0.94-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.94 - (tagged by Valentin Lorentz on 2020-02-28 14:28:11 +0100) * Upstream changes: - v0.0.94 - Allow subclasses of RPCClient to override methods. -- Software Heritage autobuilder (on jenkins-debian1) Fri, 28 Feb 2020 13:32:41 +0000 swh-core (0.0.93-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.93 - (tagged by Valentin Lorentz on 2020-02-26 15:26:29 +0100) * Upstream changes: - v0.0.93 - Reintroduce support for decoding legacy msgpack encoding -- Software Heritage autobuilder (on jenkins-debian1) Wed, 26 Feb 2020 14:29:46 +0000 swh-core (0.0.92-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.92 - (tagged by Valentin Lorentz on 2020-02-19 15:40:57 +0100) * Upstream changes: - v0.0.92 - Add support for msgpack 1.0.0 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 19 Feb 2020 14:45:27 +0000 swh-core (0.0.91-1~swh2) unstable-swh; urgency=medium * Add missing python3-iso8601 build dependency -- Antoine Lambert Wed, 19 Feb 2020 11:18:34 +0100 swh-core (0.0.91-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.91 - (tagged by Antoine Lambert on 2020-02-18 16:43:59 +0100) * Upstream changes: - version 0.0.91 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 18 Feb 2020 15:51:39 +0000 swh-core (0.0.90-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.90 - (tagged by Valentin Lorentz on 2020-02-18 13:54:30 +0100) * Upstream changes: - v0.0.90 - Remove exception pickling from the async server, as in the sync server. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 18 Feb 2020 12:59:38 +0000 swh-core (0.0.89-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.89 - (tagged by Valentin Lorentz on 2020-02-18 11:35:01 +0100) * Upstream changes: - v0.0.89 - * Change msgpack serialization to be closer to the JSON one. - * Add support for extra {de,en}coders. - * Add extra_type_encoders and extra_type_decoders attributes to RPC clients and servers. - * Use iso8601.parse_date instead of dateutil.parser.parse. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 18 Feb 2020 10:43:34 +0000 swh-core (0.0.88-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.88 - (tagged by Valentin Lorentz on 2020-02-14 12:22:23 +0100) * Upstream changes: - v0.0.88 - In case of errors, return a simple dictionary instead of pickled exception. -- Software Heritage autobuilder (on jenkins-debian1) Fri, 14 Feb 2020 11:27:02 +0000 swh-core (0.0.87-1~swh2) unstable-swh; urgency=medium * Fix package build -- Antoine R. Dumont (@ardumont) Thu, 30 Jan 2020 14:06:54 +0100 swh-core (0.0.87-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.87 - (tagged by Valentin Lorentz on 2020-01-29 12:21:48 +0100) * Upstream changes: - v0.0.87 - Make db_transaction* remove db/cur from the signature. -- Software Heritage autobuilder (on jenkins-debian1) Wed, 29 Jan 2020 11:28:23 +0000 swh-core (0.0.86-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.86 - (tagged by Antoine R. Dumont (@ardumont) on 2020-01-23 09:08:56 +0100) * Upstream changes: - v0.0.86 - sentry: Add environment variable $SWH_SENTRY_ENVIRONMENT - pytest_plugin: Fix sphinx warnings -- Software Heritage autobuilder (on jenkins-debian1) Thu, 23 Jan 2020 08:12:40 +0000 swh-core (0.0.85-1~swh2) unstable-swh; urgency=medium * Fix package build on buster -- Antoine Lambert Thu, 16 Jan 2020 11:00:00 +0000 swh-core (0.0.85-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.85 - (tagged by Valentin Lorentz on 2020-01-15 13:07:36 +0100) * Upstream changes: - v0.0.85 - Add env var SWH_MAIN_PACKAGE -- Software Heritage autobuilder (on jenkins-debian1) Wed, 15 Jan 2020 12:11:49 +0000 swh-core (0.0.84-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.84 - (tagged by Antoine R. Dumont (@ardumont) on 2019-12-13 09:08:00 +0100) * Upstream changes: - v0.0.84 - Improve tarball support for tar.lz, tar.x, tar.Z files -- Software Heritage autobuilder (on jenkins-debian1) Fri, 13 Dec 2019 08:14:23 +0000 swh-core (0.0.83-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.83 - (tagged by Antoine R. Dumont (@ardumont) on 2019-12-12 16:03:25 +0100) * Upstream changes: - v0.0.83 - core.config: Rename configuration key -- Software Heritage autobuilder (on jenkins-debian1) Thu, 12 Dec 2019 15:06:28 +0000 swh-core (0.0.82-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.82 - (tagged by Nicolas Dandrimont on 2019-12-11 15:04:09 +0100) * Upstream changes: - Release swh.core 0.0.82 - Add missing conftest.py to MANIFEST.in -- Software Heritage autobuilder (on jenkins-debian1) Wed, 11 Dec 2019 14:09:00 +0000 swh-core (0.0.81-1~swh2) unstable-swh; urgency=medium * Add dependency to python3-sentry-sdk -- Nicolas Dandrimont Wed, 11 Dec 2019 14:58:54 +0100 swh-core (0.0.81-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.81 - (tagged by Valentin Lorentz on 2019-12-10 13:59:17 +0100) * Upstream changes: - v0.0.81 - * Include all requirements in MANIFEST.in - * Split test requirements to try and properly minimize dependencies - * Make the CLI initialize sentry-sdk based on CLI options/envvars. - * Add gunicorn config script to initialize sentry-sdk based on envvars. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 10 Dec 2019 13:03:13 +0000 swh-core (0.0.80-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.80 - (tagged by Nicolas Dandrimont on 2019-11-19 16:36:35 +0100) * Upstream changes: - Release swh.core v0.0.80 - Let TypeErrors pass through the RPC layer - Register SIGINT/SIGTERM handlers for the CLI -- Software Heritage autobuilder (on jenkins-debian1) Tue, 19 Nov 2019 15:41:09 +0000 swh-core (0.0.79-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.79 - (tagged by Stefano Zacchiroli on 2019-11-18 13:35:19 +0100) * Upstream changes: - v0.0.79 ======= - * RPCClient: add response attribute to RemoteException - * RPCClent: rename and refactor check_status (now raise_for_status) - * RPCClient: check HTTP status code for errors also when streaming - * cli: Add support for loading a logging configuration file - * cli: Allow adding a Notes section between Options and Commands - * Add trailing dot to help texts for consistency - * logger: only flatten dicts if all keys are strings - * Move to @pytest.fixture from yield_fixture - * test_rpc_client_server.py: fix typo in docstring -- Software Heritage autobuilder (on jenkins-debian1) Mon, 18 Nov 2019 12:42:12 +0000 swh-core (0.0.78-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.78 - (tagged by Nicolas Dandrimont on 2019-11-06 18:01:56 +0100) * Upstream changes: - Release swh.core 0.0.78 - allow the swh command to work even when a plugin fails - hardcode bytea and bytea[] type oids in BaseDb -- Software Heritage autobuilder (on jenkins-debian1) Wed, 06 Nov 2019 17:05:38 +0000 swh-core (0.0.77-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.77 - (tagged by Antoine R. Dumont (@ardumont) on 2019-11-06 14:10:58 +0100) * Upstream changes: - v0.0.77 - pytest_plugin: Decode url to resolve filename - api/serializers: Force json module use to decode requests text response -- Software Heritage autobuilder (on jenkins-debian1) Wed, 06 Nov 2019 13:15:03 +0000 swh-core (0.0.76-1~swh2) unstable-swh; urgency=medium * Force using the swh.core pytest plugin -- Nicolas Dandrimont Wed, 23 Oct 2019 14:50:04 +0200 swh-core (0.0.76-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.76 - (tagged by Nicolas Dandrimont on 2019-10-18 10:16:20 +0200) * Upstream changes: - Release swh.core v0.0.76 - Make the systemd dependency optional -- Software Heritage autobuilder (on jenkins-debian1) Fri, 18 Oct 2019 08:25:15 +0000 swh-core (0.0.75-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.75 - (tagged by Antoine R. Dumont (@ardumont) on 2019-10-14 17:51:58 +0200) * Upstream changes: - v0.0.75 - pytest_plugin: Add support for http requests -- Software Heritage autobuilder (on jenkins-debian1) Mon, 14 Oct 2019 15:57:05 +0000 swh-core (0.0.74-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.74 - (tagged by David Douard on 2019-10-11 15:30:51 +0200) * Upstream changes: - v0.0.74 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 11 Oct 2019 13:35:17 +0000 swh-core (0.0.73-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.73 - (tagged by Antoine R. Dumont (@ardumont) on 2019-10-09 14:16:04 +0200) * Upstream changes: - v0.0.73 - Improve pytest-plugin fixture to ease testing with pagination -- Software Heritage autobuilder (on jenkins-debian1) Wed, 09 Oct 2019 12:20:36 +0000 swh-core (0.0.72-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.72 - (tagged by Antoine R. Dumont (@ardumont) on 2019-10-09 10:59:28 +0200) * Upstream changes: - v0.0.72 - Fix tox.ini's py3 environment -- Software Heritage autobuilder (on jenkins-debian1) Wed, 09 Oct 2019 09:02:51 +0000 swh-core (0.0.70-1~swh2) unstable-swh; urgency=medium * Add new dependency on python3-tz -- Nicolas Dandrimont Tue, 01 Oct 2019 15:07:09 +0200 swh-core (0.0.70-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.70 - (tagged by Stefano Zacchiroli on 2019-09-27 10:16:41 +0200) * Upstream changes: - v0.0.70 - init.py: switch to documented way of extending path -- Software Heritage autobuilder (on jenkins-debian1) Fri, 27 Sep 2019 08:21:29 +0000 swh-core (0.0.69-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.69 - (tagged by Stefano Zacchiroli on 2019-09-20 15:50:52 +0200) * Upstream changes: - v0.0.69 - MANIFEST.in: ship py.typed -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 Sep 2019 13:54:15 +0000 swh-core (0.0.68-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.68 - (tagged by Stefano Zacchiroli on 2019-09-20 15:05:29 +0200) * Upstream changes: - v0.0.68 - * mypy: ignore django-stubs, needed only by hypothesis - * mypy: use conffile to ignore requests_mock - * typing: minimal changes to make a no-op mypy run pass - * db_testing.py: do not explode when TEST_DB_DUMP = None - * swh.core.config.parse_config_file: fix sphinx markup in docstring - * statsd: protect access to the statsd's socket - * tests: add tests for swh.logger and swh.tarball modules - * Remove fallback when aiohttp_utils is not installed. -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 Sep 2019 13:09:50 +0000 swh-core (0.0.67-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.67 - (tagged by Valentin Lorentz on 2019-08-22 13:56:36 +0200) * Upstream changes: - v0.0.67 - Improve error handling in Db.copy_to -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Aug 2019 12:02:14 +0000 swh-core (0.0.66-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.66 - (tagged by David Douard on 2019-07-30 13:55:16 +0200) * Upstream changes: - v0.0.66 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 30 Jul 2019 11:58:47 +0000 swh-core (0.0.65-1~swh2) unstable-swh; urgency=medium * debian/control: add missing dependencies. -- David Douard Tue, 16 Jul 2019 14:46:43 +0200 swh-core (0.0.65-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.65 - (tagged by David Douard on 2019-07-15 16:49:47 +0200) * Upstream changes: - v0.0.65 - needed to fix my mess with 0.0.64 tag, since the wrong 0.0.64 version has - already been pushed to pypi. -- Software Heritage autobuilder (on jenkins-debian1) Mon, 15 Jul 2019 14:53:29 +0000 swh-core (0.0.64-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.64 - (tagged by David Douard on 2019-07-15 16:33:32 +0200) * Upstream changes: - v0.0.64 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 15 Jul 2019 14:37:04 +0000 swh-core (0.0.63-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.63 - (tagged by Antoine Lambert on 2019-05-21 13:12:11 +0200) * Upstream changes: - version 0.0.63 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 21 May 2019 11:15:45 +0000 swh-core (0.0.62-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.62 - (tagged by Antoine Lambert on 2019-05-20 14:56:05 +0200) * Upstream changes: - version 0.0.62 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 20 May 2019 13:01:38 +0000 swh-core (0.0.61-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.61 - (tagged by David Douard on 2019-05-17 10:32:07 +0200) * Upstream changes: - v0.0.61 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 17 May 2019 08:38:08 +0000 swh-core (0.0.60-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.60 - (tagged by David Douard on 2019-05-06 15:27:44 +0200) * Upstream changes: - v0.0.60 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 06 May 2019 13:32:48 +0000 swh-core (0.0.59-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.59 - (tagged by Valentin Lorentz on 2019-04-09 16:55:41 +0200) * Upstream changes: - Explicitly give Db connections back to the pool. - So they gracefully release the connection on error instead - of relying on reference-counting to call the Db's `__del__` - (which does not happen in Hypothesis tests) because a ref - to it is kept via the traceback object. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 09 Apr 2019 16:12:32 +0000 swh-core (0.0.58-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.58 - (tagged by Antoine Lambert on 2019-04-02 17:19:05 +0200) * Upstream changes: - version 0.0.58 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 02 Apr 2019 15:24:34 +0000 swh-core (0.0.57-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.57 - (tagged by Nicolas Dandrimont on 2019-03-28 15:51:27 +0100) * Upstream changes: - Release swh.core v0.0.57 - Move to native async primitives - Fix statsd.timed exceptional behavior bug/misfeature - Fix SWHRemoteAPI post_stream method -- Software Heritage autobuilder (on jenkins-debian1) Thu, 28 Mar 2019 14:55:58 +0000 swh-core (0.0.56-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.56 - (tagged by David Douard on 2019-03-19 10:17:06 +0100) * Upstream changes: - v0.0.56 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 19 Mar 2019 09:27:18 +0000 swh-core (0.0.55-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.55 - (tagged by Antoine R. Dumont (@ardumont) on 2019-02-19 12:28:26 +0100) * Upstream changes: - v0.0.55 - Fix runtime dependencies -- Software Heritage autobuilder (on jenkins-debian1) Tue, 19 Feb 2019 11:32:28 +0000 swh-core (0.0.54-1~swh2) unstable-swh; urgency=medium * New upstream release 0.0.54 * Upstream changes: - Add missing build dependencies -- Antoine R. Dumont (@ardumont) Tue, 12 Feb 2019 16:25:34 +0000 swh-core (0.0.54-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.54 - (tagged by Valentin Lorentz on 2019-02-11 16:47:18 +0100) * Upstream changes: - Add test for BaseDb.connect. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 12 Feb 2019 12:37:43 +0000 swh-core (0.0.53-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.53 - (tagged by Antoine R. Dumont (@ardumont) on 2019-02-08 09:09:30 +0100) * Upstream changes: - v0.0.53 - Fix debian build -- Software Heritage autobuilder (on jenkins-debian1) Fri, 08 Feb 2019 08:12:31 +0000 swh-core (0.0.52-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.52 - (tagged by David Douard on 2019-02-06 15:24:04 +0100) * Upstream changes: - v0.0.52 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 06 Feb 2019 14:27:14 +0000 swh-core (0.0.51-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.51 - (tagged by David Douard on 2019-02-01 14:28:27 +0100) * Upstream changes: - v0.0.51 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 01 Feb 2019 13:31:45 +0000 swh-core (0.0.50-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.50 - (tagged by Nicolas Dandrimont on 2019-01-09 15:50:58 +0100) * Upstream changes: - Release swh.core v0.0.50 - Add statsd client module - Log used config files -- Software Heritage autobuilder (on jenkins-debian1) Wed, 09 Jan 2019 14:54:37 +0000 swh-core (0.0.49-1~swh1) unstable-swh; urgency=medium * Make DbTestFixture.setUp() accept and pass *args and **kwargs. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 08 Jan 2019 16:38:02 +0000 swh-core (0.0.48-1~swh1) unstable-swh; urgency=medium * v0.0.48 * swh.core.cli: Update swh-db-init to make it idemtpotent -- Antoine R. Dumont (@ardumont) Tue, 08 Jan 2019 15:33:15 +0000 swh-core (0.0.47-1~swh1) unstable-swh; urgency=medium * v0.0.47 * swh.core.cli: Fix flag -- Antoine R. Dumont (@ardumont) Tue, 08 Jan 2019 15:16:09 +0000 swh-core (0.0.46-1~swh1) unstable-swh; urgency=medium * v0.0.46 * utils.grouper: Improve implementation * Remove now-obsolete information about swh.core.worker -- Antoine R. Dumont (@ardumont) Tue, 08 Jan 2019 14:37:34 +0000 swh-core (0.0.45-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.45 * Compatibility with recent msgpack * Debian packaging-related cleanups -- Nicolas Dandrimont Thu, 22 Nov 2018 21:09:53 +0100 swh-core (0.0.44-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.44 * Refactor the database testing fixtures * Stop unsafe serialization/deserialization constructs * Update tests to use nose -- Nicolas Dandrimont Thu, 18 Oct 2018 18:20:12 +0200 swh-core (0.0.43-1~swh1) unstable-swh; urgency=medium * v0.0.43 * Fix missing dependency declaration -- Antoine R. Dumont (@ardumont) Thu, 11 Oct 2018 15:47:06 +0200 swh-core (0.0.42-1~swh1) unstable-swh; urgency=medium * v0.0.42 * Fix missing dependency declaration -- Antoine R. Dumont (@ardumont) Thu, 11 Oct 2018 15:45:25 +0200 swh-core (0.0.41-1~swh1) unstable-swh; urgency=medium * Add functions to generate HTTP API clients and servers from databases. * Summary: This moves the interesting parts of D505 into the core, so other components can use them as well. * Test Plan: `make test` * Reviewers: ardumont, seirl, #reviewers * Reviewed By: ardumont, #reviewers * Subscribers: douardda * Differential Revision: https://forge.softwareheritage.org/D507 -- Valentin Lorentz Thu, 11 Oct 2018 10:57:27 +0200 swh-core (0.0.40-1~swh1) unstable-swh; urgency=medium * v0.0.40 * swh.core.api.SWHRemoteAPI: Permit to set a query timeout option -- Antoine R. Dumont (@ardumont) Thu, 24 May 2018 12:10:03 +0200 swh-core (0.0.39-1~swh1) unstable-swh; urgency=medium * v0.0.39 * package: Add missing runtime dependency -- Antoine R. Dumont (@ardumont) Thu, 26 Apr 2018 15:24:22 +0200 swh-core (0.0.38-1~swh1) unstable-swh; urgency=medium * v0.0.38 * tests: Use more reasonable psql options for db restores * swh.core.serializers: Add custom types serialization -- Antoine R. Dumont (@ardumont) Thu, 26 Apr 2018 15:15:27 +0200 swh-core (0.0.37-1~swh1) unstable-swh; urgency=medium * v0.0.37 * Move test fixture in swh.core.tests.server_testing module -- Antoine R. Dumont (@ardumont) Wed, 25 Apr 2018 15:00:02 +0200 swh-core (0.0.36-1~swh1) unstable-swh; urgency=medium * v0.0.36 * Migrate swh.loader.tar.tarball module in swh.core -- Antoine R. Dumont (@ardumont) Wed, 06 Dec 2017 12:03:29 +0100 swh-core (0.0.35-1~swh1) unstable-swh; urgency=medium * Release swh.core version 0.0.35 * Update packaging runes -- Nicolas Dandrimont Thu, 12 Oct 2017 18:07:50 +0200 swh-core (0.0.34-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.34 * New modular database test fixture -- Nicolas Dandrimont Mon, 07 Aug 2017 18:29:48 +0200 swh-core (0.0.33-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.33 * Be more conservative with remote API responses -- Nicolas Dandrimont Mon, 19 Jun 2017 19:01:38 +0200 swh-core (0.0.32-1~swh1) unstable-swh; urgency=medium * Release swh-core v0.0.32 * Add asynchronous streaming methods for internal APIs * Remove task arguments from systemd-journal loggers -- Nicolas Dandrimont Tue, 09 May 2017 14:04:22 +0200 swh-core (0.0.31-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.31 * Add explicit dependency on python3-systemd -- Nicolas Dandrimont Fri, 07 Apr 2017 15:11:26 +0200 swh-core (0.0.30-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.30 * drop swh.core.hashutil (moved to swh.model.hashutil) * add a systemd logger -- Nicolas Dandrimont Fri, 07 Apr 2017 11:49:15 +0200 swh-core (0.0.29-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.29 * Catch proper exception in the base API client -- Nicolas Dandrimont Thu, 02 Feb 2017 00:19:25 +0100 swh-core (0.0.28-1~swh1) unstable-swh; urgency=medium * v0.0.28 * Refactoring some common code into swh.core -- Antoine R. Dumont (@ardumont) Thu, 26 Jan 2017 14:54:22 +0100 swh-core (0.0.27-1~swh1) unstable-swh; urgency=medium * v0.0.27 * Fix issue with default boolean value -- Antoine R. Dumont (@ardumont) Thu, 20 Oct 2016 16:15:20 +0200 swh-core (0.0.26-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.26 * Raise an exception when a configuration file exists and is unreadable -- Nicolas Dandrimont Wed, 12 Oct 2016 10:16:09 +0200 swh-core (0.0.25-1~swh1) unstable-swh; urgency=medium * v0.0.25 * Add new function utils.cwd -- Antoine R. Dumont (@ardumont) Thu, 29 Sep 2016 21:29:37 +0200 swh-core (0.0.24-1~swh1) unstable-swh; urgency=medium * v0.0.24 * Deal with edge case in logger regarding json -- Antoine R. Dumont (@ardumont) Thu, 22 Sep 2016 12:21:09 +0200 swh-core (0.0.23-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.23 * Properly fix the PyYAML dependency -- Nicolas Dandrimont Tue, 23 Aug 2016 16:20:29 +0200 swh-core (0.0.22-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.22 * Proper loading of yaml and ini files in all paths -- Nicolas Dandrimont Fri, 19 Aug 2016 15:45:55 +0200 swh-core (0.0.21-1~swh1) unstable-swh; urgency=medium * v0.0.21 * Update test tools -- Antoine R. Dumont (@ardumont) Tue, 19 Jul 2016 14:47:01 +0200 swh-core (0.0.20-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.20 * Add some generic bytes <-> escaped unicode methods -- Nicolas Dandrimont Tue, 14 Jun 2016 16:54:41 +0200 swh-core (0.0.19-1~swh1) unstable-swh; urgency=medium * v0.0.19 * Resurrect swh.core.utils -- Antoine R. Dumont (@ardumont) Fri, 15 Apr 2016 12:40:43 +0200 swh-core (0.0.18-1~swh1) unstable-swh; urgency=medium * v0.0.18 * Add swh.core.utils * serializers: support UUIDs all around -- Antoine R. Dumont (@ardumont) Sat, 26 Mar 2016 11:16:33 +0100 swh-core (0.0.17-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.17 * Allow serialization of UUIDs -- Nicolas Dandrimont Fri, 04 Mar 2016 11:40:56 +0100 swh-core (0.0.16-1~swh1) unstable-swh; urgency=medium * Release swh.core version 0.0.16 * add bytehex_to_hash and hash_to_bytehex in hashutil * move scheduling utilities to swh.scheduler -- Nicolas Dandrimont Fri, 19 Feb 2016 18:12:10 +0100 swh-core (0.0.15-1~swh1) unstable-swh; urgency=medium * Release v0.0.15 * Add hashutil.hash_git_object -- Nicolas Dandrimont Wed, 16 Dec 2015 16:31:26 +0100 swh-core (0.0.14-1~swh1) unstable-swh; urgency=medium * v0.0.14 * Add simple README * Update license * swh.core.hashutil.hashfile can now deal with filepath as bytes -- Antoine R. Dumont (@ardumont) Fri, 23 Oct 2015 11:13:14 +0200 swh-core (0.0.13-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.13 -- Nicolas Dandrimont Fri, 09 Oct 2015 17:32:49 +0200 swh-core (0.0.12-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.12 -- Nicolas Dandrimont Tue, 06 Oct 2015 17:34:34 +0200 swh-core (0.0.11-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.11 -- Nicolas Dandrimont Sat, 03 Oct 2015 15:57:03 +0200 swh-core (0.0.10-1~swh1) unstable-swh; urgency=medium * Prepare deploying swh.core v0.0.10 -- Nicolas Dandrimont Sat, 03 Oct 2015 12:28:52 +0200 swh-core (0.0.9-1~swh1) unstable-swh; urgency=medium * Prepare deploying swh.core v0.0.9 -- Nicolas Dandrimont Sat, 03 Oct 2015 11:36:55 +0200 swh-core (0.0.8-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.8 -- Nicolas Dandrimont Thu, 01 Oct 2015 12:31:44 +0200 swh-core (0.0.7-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.7 -- Nicolas Dandrimont Thu, 01 Oct 2015 11:29:04 +0200 swh-core (0.0.6-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.6 -- Nicolas Dandrimont Tue, 29 Sep 2015 16:48:44 +0200 swh-core (0.0.5-1~swh1) unstable-swh; urgency=medium * Prepare v0.0.5 deployment -- Nicolas Dandrimont Tue, 29 Sep 2015 16:08:32 +0200 swh-core (0.0.4-1~swh1) unstable-swh; urgency=medium * Tagging swh.core 0.0.4 -- Nicolas Dandrimont Fri, 25 Sep 2015 15:41:26 +0200 swh-core (0.0.3-1~swh1) unstable-swh; urgency=medium * Tag swh.core v0.0.3 -- Nicolas Dandrimont Fri, 25 Sep 2015 11:07:10 +0200 swh-core (0.0.2-1~swh1) unstable-swh; urgency=medium * Deploy v0.0.2 -- Nicolas Dandrimont Wed, 23 Sep 2015 12:08:50 +0200 swh-core (0.0.1-1~swh1) unstable-swh; urgency=medium * Initial release * Tag v0.0.1 for deployment -- Nicolas Dandrimont Tue, 22 Sep 2015 14:52:26 +0200 diff --git a/swh.core.egg-info/PKG-INFO b/swh.core.egg-info/PKG-INFO index 376f27c..045f03f 100644 --- a/swh.core.egg-info/PKG-INFO +++ b/swh.core.egg-info/PKG-INFO @@ -1,39 +1,39 @@ Metadata-Version: 2.1 Name: swh.core -Version: 2.15.0 +Version: 2.16.0 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-core Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-core/ Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/x-rst Provides-Extra: testing-core Provides-Extra: logging Provides-Extra: db Provides-Extra: http Provides-Extra: github Provides-Extra: testing License-File: LICENSE License-File: AUTHORS Software Heritage - Core foundations ==================================== Low-level utilities and helpers used by almost all other modules in the stack. core library for swh's modules: - config parser - serialization - logging mechanism - database connection - http-based RPC client/server diff --git a/swh/core/tarball.py b/swh/core/tarball.py index b248b31..c80cb5c 100644 --- a/swh/core/tarball.py +++ b/swh/core/tarball.py @@ -1,249 +1,260 @@ # Copyright (C) 2015-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import shutil import stat from subprocess import run import tarfile import zipfile import magic from . import utils MIMETYPE_TO_ARCHIVE_FORMAT = { "application/x-compress": "tar.Z|x", "application/x-tar": "tar", "application/x-bzip2": "bztar", "application/gzip": "gztar", + "application/x-gzip": "gztar", "application/x-lzip": "tar.lz", "application/zip": "zip", "application/java-archive": "jar", "application/zstd": "tar.zst", "application/x-zstd": "tar.zst", } def _unpack_tar(tarpath: str, extract_dir: str) -> str: """Unpack tarballs unsupported by the standard python library. Examples include tar.Z, tar.lz, tar.x, etc.... As this implementation relies on the `tar` command, this function supports the same compression the tar command supports. This expects the `extract_dir` to exist. Raises: shutil.ReadError in case of issue uncompressing the archive (tarpath does not exist, extract_dir does not exist, etc...) Returns: full path to the uncompressed directory. """ try: run(["tar", "xf", tarpath, "-C", extract_dir], check=True) return extract_dir except Exception as e: raise shutil.ReadError( f"Unable to uncompress {tarpath} to {extract_dir}. Reason: {e}" ) def _unpack_zip(zippath: str, extract_dir: str) -> str: """Unpack zip files unsupported by the standard python library, for instance those with legacy compression type 6 (implode). This expects the `extract_dir` to exist. Raises: shutil.ReadError in case of issue uncompressing the archive (zippath does not exist, extract_dir does not exist, etc...) Returns: full path to the uncompressed directory. """ try: run(["unzip", "-q", "-d", extract_dir, zippath], check=True) return extract_dir except Exception as e: raise shutil.ReadError( f"Unable to uncompress {zippath} to {extract_dir}. Reason: {e}" ) def _unpack_jar(jarpath: str, extract_dir: str) -> str: """Unpack jar files using standard Python module zipfile. This expects the `extract_dir` to exist. Raises: shutil.ReadError in case of issue uncompressing the archive (jarpath does not exist, extract_dir does not exist, etc...) Returns: full path to the uncompressed directory. """ try: with zipfile.ZipFile(jarpath) as jar: jar.extractall(path=extract_dir) return extract_dir except Exception as e: raise shutil.ReadError( f"Unable to uncompress {jarpath} to {extract_dir}. Reason: {e}" ) def _unpack_zst(zstpath: str, extract_dir: str) -> str: """Unpack zst files unsupported by the standard python library. Example include tar.zst This expects the `extract_dir` to exist. Raises: shutil.ReadError in case of issue uncompressing the archive (zstpath """ try: run( ["tar", "--force-local", "-I 'zstd'", "-xf", zstpath, "-C", extract_dir], check=True, ) return extract_dir except Exception as e: raise shutil.ReadError( f"Unable to uncompress {zstpath} to {extract_dir}. Reason: {e}" ) def register_new_archive_formats(): """Register new archive formats to uncompress""" registered_formats = [f[0] for f in shutil.get_unpack_formats()] for name, extensions, function in ADDITIONAL_ARCHIVE_FORMATS: if name in registered_formats: continue shutil.register_unpack_format(name, extensions, function) def uncompress(tarpath: str, dest: str): """Uncompress tarpath to dest folder if tarball is supported. Note that this fixes permissions after successfully uncompressing the archive. Args: tarpath: path to tarball to uncompress dest: the destination folder where to uncompress the tarball, it will be created if it does not exist Raises: ValueError when a problem occurs during unpacking """ try: os.makedirs(dest, exist_ok=True) format = None # try to get archive format from extension for format_, exts, _ in shutil.get_unpack_formats(): if any([tarpath.lower().endswith(ext.lower()) for ext in exts]): format = format_ break # try to get archive format from file mimetype if format is None: m = magic.Magic(mime=True) mime = m.from_file(tarpath) format = MIMETYPE_TO_ARCHIVE_FORMAT.get(mime) shutil.unpack_archive(tarpath, extract_dir=dest, format=format) except shutil.ReadError as e: raise ValueError(f"Problem during unpacking {tarpath}. Reason: {e}") except NotImplementedError: if tarpath.lower().endswith(".zip") or format == "zip": _unpack_zip(tarpath, dest) else: raise + except NotADirectoryError: + if format and "tar" in format: + # some old tarballs might fail to be unpacked by shutil.unpack_archive, + # fallback using the tar command as last resort + _unpack_tar(tarpath, dest) + else: + raise normalize_permissions(dest) def normalize_permissions(path: str): """Normalize the permissions of all files and directories under `path`. This makes all subdirectories and files with the user executable bit set mode 0o0755, and all other files mode 0o0644. Args: path: the path under which permissions should be normalized """ - for dirpath, _, fnames in os.walk(path): - os.chmod(dirpath, 0o0755) + os.chmod(path, 0o0755) + for dirpath, dnames, fnames in os.walk(path): + for dname in dnames: + dpath = os.path.join(dirpath, dname) + os.chmod(dpath, 0o0755) for fname in fnames: fpath = os.path.join(dirpath, fname) if not os.path.islink(fpath): is_executable = os.stat(fpath).st_mode & stat.S_IXUSR forced_mode = 0o0755 if is_executable else 0o0644 os.chmod(fpath, forced_mode) def _ls(rootdir): """Generator of filepath, filename from rootdir.""" for dirpath, dirnames, fnames in os.walk(rootdir): for fname in dirnames + fnames: fpath = os.path.join(dirpath, fname) fname = utils.commonname(rootdir, fpath) yield fpath, fname def _compress_zip(tarpath, files): """Compress dirpath's content as tarpath.""" with zipfile.ZipFile(tarpath, "w") as z: for fpath, fname in files: z.write(fpath, arcname=fname) def _compress_tar(tarpath, files): """Compress dirpath's content as tarpath.""" with tarfile.open(tarpath, "w:bz2") as t: for fpath, fname in files: t.add(fpath, arcname=fname, recursive=False) def compress(tarpath, nature, dirpath_or_files): """Create a tarball tarpath with nature nature. The content of the tarball is either dirpath's content (if representing a directory path) or dirpath's iterable contents. Compress the directory dirpath's content to a tarball. The tarball being dumped at tarpath. The nature of the tarball is determined by the nature argument. """ if isinstance(dirpath_or_files, str): files = _ls(dirpath_or_files) else: # iterable of 'filepath, filename' files = dirpath_or_files if nature == "zip": _compress_zip(tarpath, files) else: _compress_tar(tarpath, files) return tarpath # Additional uncompression archive format support ADDITIONAL_ARCHIVE_FORMATS = [ # name, extensions, function ("tar.Z|x", [".tar.Z", ".tar.x"], _unpack_tar), ("jar", [".jar"], _unpack_jar), ("tbz2", [".tbz", "tbz2"], _unpack_tar), # FIXME: make this optional depending on the runtime lzip package install ("tar.lz", [".tar.lz"], _unpack_tar), ("crate", [".crate"], _unpack_tar), ("tar.zst", [".tar.zst", ".tar.zstd"], _unpack_zst), ] register_new_archive_formats() diff --git a/swh/core/tests/test_tarball.py b/swh/core/tests/test_tarball.py index 366ca5a..0231a2a 100644 --- a/swh/core/tests/test_tarball.py +++ b/swh/core/tests/test_tarball.py @@ -1,254 +1,284 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import hashlib +import io import os import shutil +import stat +import tarfile import pytest from swh.core import tarball @pytest.fixture def prepare_shutil_state(): """Reset any shutil modification in its current state""" import shutil registered_formats = [f[0] for f in shutil.get_unpack_formats()] for format_id in tarball.ADDITIONAL_ARCHIVE_FORMATS: name = format_id[0] if name in registered_formats: shutil.unregister_unpack_format(name) return shutil def test_compress_uncompress_zip(tmp_path): tocompress = tmp_path / "compressme" tocompress.mkdir() for i in range(10): fpath = tocompress / ("file%s.txt" % i) fpath.write_text("content of file %s" % i) zipfile = tmp_path / "archive.zip" tarball.compress(str(zipfile), "zip", str(tocompress)) destdir = tmp_path / "destdir" tarball.uncompress(str(zipfile), str(destdir)) lsdir = sorted(x.name for x in destdir.iterdir()) assert ["file%s.txt" % i for i in range(10)] == lsdir @pytest.mark.xfail( reason=( "Python's zipfile library doesn't support Info-ZIP's " "extension for file permissions." ) ) def test_compress_uncompress_zip_modes(tmp_path): tocompress = tmp_path / "compressme" tocompress.mkdir() fpath = tocompress / "text.txt" fpath.write_text("echo foo") fpath.chmod(0o644) fpath = tocompress / "executable.sh" fpath.write_text("echo foo") fpath.chmod(0o755) zipfile = tmp_path / "archive.zip" tarball.compress(str(zipfile), "zip", str(tocompress)) destdir = tmp_path / "destdir" tarball.uncompress(str(zipfile), str(destdir)) (executable_path, text_path) = sorted(destdir.iterdir()) assert text_path.stat().st_mode == 0o100644 # succeeds, it's the default assert executable_path.stat().st_mode == 0o100755 # fails def test_compress_uncompress_tar(tmp_path): tocompress = tmp_path / "compressme" tocompress.mkdir() for i in range(10): fpath = tocompress / ("file%s.txt" % i) fpath.write_text("content of file %s" % i) tarfile = tmp_path / "archive.tar" tarball.compress(str(tarfile), "tar", str(tocompress)) destdir = tmp_path / "destdir" tarball.uncompress(str(tarfile), str(destdir)) lsdir = sorted(x.name for x in destdir.iterdir()) assert ["file%s.txt" % i for i in range(10)] == lsdir def test_compress_uncompress_tar_modes(tmp_path): tocompress = tmp_path / "compressme" tocompress.mkdir() fpath = tocompress / "text.txt" fpath.write_text("echo foo") fpath.chmod(0o644) fpath = tocompress / "executable.sh" fpath.write_text("echo foo") fpath.chmod(0o755) tarfile = tmp_path / "archive.tar" tarball.compress(str(tarfile), "tar", str(tocompress)) destdir = tmp_path / "destdir" tarball.uncompress(str(tarfile), str(destdir)) (executable_path, text_path) = sorted(destdir.iterdir()) assert text_path.stat().st_mode == 0o100644 assert executable_path.stat().st_mode == 0o100755 def test_uncompress_tar_failure(tmp_path, datadir): """Unpack inexistent tarball should fail""" tarpath = os.path.join(datadir, "archives", "inexistent-archive.tar.Z") assert not os.path.exists(tarpath) with pytest.raises(ValueError, match="Problem during unpacking"): tarball.uncompress(tarpath, tmp_path) def test_uncompress_tar(tmp_path, datadir): """Unpack supported tarball into an existent folder should be ok""" filename = "groff-1.02.tar.Z" tarpath = os.path.join(datadir, "archives", filename) assert os.path.exists(tarpath) extract_dir = os.path.join(tmp_path, filename) tarball.uncompress(tarpath, extract_dir) assert len(os.listdir(extract_dir)) > 0 def test_uncompress_tarzst(tmp_path, datadir): """Unpack supported zst tarball into an existent folder should be ok""" filename = "ca-certificates-20210603-1-any.pkg.tar.zst" tarpath = os.path.join(datadir, "archives", filename) assert os.path.exists(tarpath) extract_dir = os.path.join(tmp_path, filename) tarball.uncompress(tarpath, extract_dir) assert len(os.listdir(extract_dir)) > 0 def test_register_new_archive_formats(prepare_shutil_state): """Registering new archive formats should be fine""" unpack_formats_v1 = [f[0] for f in shutil.get_unpack_formats()] for format_id in tarball.ADDITIONAL_ARCHIVE_FORMATS: assert format_id[0] not in unpack_formats_v1 # when tarball.register_new_archive_formats() # then unpack_formats_v2 = [f[0] for f in shutil.get_unpack_formats()] for format_id in tarball.ADDITIONAL_ARCHIVE_FORMATS: assert format_id[0] in unpack_formats_v2 def test_uncompress_archives(tmp_path, datadir): """High level call uncompression on supported archives""" archive_dir = os.path.join(datadir, "archives") archive_files = os.listdir(archive_dir) for archive_file in archive_files: archive_path = os.path.join(archive_dir, archive_file) extract_dir = os.path.join(tmp_path, archive_file) tarball.uncompress(archive_path, dest=extract_dir) assert len(os.listdir(extract_dir)) > 0 def test_normalize_permissions(tmp_path): for perms in range(0o1000): filename = str(perms) file_path = tmp_path / filename file_path.touch() file_path.chmod(perms) + # add directory without any permission + dir_path = tmp_path / "dir" + dir_path.mkdir(mode=stat.S_IFDIR) + for file in tmp_path.iterdir(): - assert file.stat().st_mode == 0o100000 | int(file.name) + if file.is_dir(): + assert file.stat().st_mode == stat.S_IFDIR + else: + assert file.stat().st_mode == 0o100000 | int(file.name) tarball.normalize_permissions(str(tmp_path)) for file in tmp_path.iterdir(): - if int(file.name) & 0o100: # original file was executable for its owner - assert file.stat().st_mode == 0o100755 + if file.is_dir(): + assert file.stat().st_mode == stat.S_IFDIR + 0o000755 else: - assert file.stat().st_mode == 0o100644 + if int(file.name) & 0o100: # original file was executable for its owner + assert file.stat().st_mode == 0o100755 + else: + assert file.stat().st_mode == 0o100644 -def test_unpcompress_zip_imploded(tmp_path, datadir): - """Unpack a zip archive with compression type 6 (implode), - not supported by python zipfile module. - - """ - filename = "msk316src.zip" - zippath = os.path.join(datadir, "archives", filename) - - assert os.path.exists(zippath) +def uncompress_archive_test(archive_path, tmp_path): + assert os.path.exists(archive_path) - extract_dir = os.path.join(tmp_path, filename) + extract_dir = os.path.join(tmp_path, "unpack", os.path.basename(archive_path)) + tarball.uncompress(archive_path, extract_dir) + assert len(os.listdir(extract_dir)) > 0 - tarball.uncompress(zippath, extract_dir) - assert len(os.listdir(extract_dir)) > 0 +def test_uncompress_tar_fallback_when_unpack_archive_failed(tmp_path): + """shutil.unpack_archive can fail unpacking a tarball with missing + directory bit set on folder contained in it while the tar command + succeeds to perform the same task so check the fallback of using + tar command works for such edge case. + """ + archive_path = os.path.join(tmp_path, "repro.tar.gz") + tf = tarfile.open(archive_path, "w:gz") + ti = tarfile.TarInfo("dir") + ti.mode = 0o777 + ti.type = tarfile.DIRTYPE + tf.addfile(ti) + ti = tarfile.TarInfo("dir/file") + tf.addfile(ti, io.BytesIO(b"hello world")) + tf.close() + uncompress_archive_test(archive_path, tmp_path) + + +def test_uncompress_zip_imploded(tmp_path, datadir): + """Unpack a zip archive with compression type 6 (implode), + not supported by python zipfile module. + """ + archive_path = os.path.join(datadir, "archives", "msk316src.zip") + uncompress_archive_test(archive_path, tmp_path) def test_uncompress_upper_archive_extension(tmp_path, datadir): """Copy test archives in a temporary directory but turn their names to uppercase, then check they can be successfully extracted. """ archives_path = os.path.join(datadir, "archives") archive_files = [ f for f in os.listdir(archives_path) if os.path.isfile(os.path.join(archives_path, f)) ] for archive_file in archive_files: archive_file_upper = os.path.join(tmp_path, archive_file.upper()) extract_dir = os.path.join(tmp_path, archive_file) shutil.copy(os.path.join(archives_path, archive_file), archive_file_upper) tarball.uncompress(archive_file_upper, extract_dir) assert len(os.listdir(extract_dir)) > 0 def test_uncompress_archive_no_extension(tmp_path, datadir): """Copy test archives in a temporary directory but turn their names to their md5 sums, then check they can be successfully extracted. """ archives_path = os.path.join(datadir, "archives") archive_files = [ f for f in os.listdir(archives_path) if os.path.isfile(os.path.join(archives_path, f)) ] for archive_file in archive_files: archive_file_path = os.path.join(archives_path, archive_file) with open(archive_file_path, "rb") as f: md5sum = hashlib.md5(f.read()).hexdigest() archive_file_md5sum = os.path.join(tmp_path, md5sum) extract_dir = os.path.join(tmp_path, archive_file) shutil.copy(archive_file_path, archive_file_md5sum) tarball.uncompress(archive_file_md5sum, extract_dir) assert len(os.listdir(extract_dir)) > 0