diff --git a/PKG-INFO b/PKG-INFO index 6e7de3a..f08d69f 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,39 +1,39 @@ Metadata-Version: 2.1 Name: swh.core -Version: 2.13.1 +Version: 2.14.0 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-core Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-core/ Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/x-rst Provides-Extra: testing-core Provides-Extra: logging Provides-Extra: db Provides-Extra: http Provides-Extra: github Provides-Extra: testing License-File: LICENSE License-File: AUTHORS Software Heritage - Core foundations ==================================== Low-level utilities and helpers used by almost all other modules in the stack. core library for swh's modules: - config parser - serialization - logging mechanism - database connection - http-based RPC client/server diff --git a/debian/changelog b/debian/changelog index 186f3b4..2df8b94 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,1435 +1,1440 @@ -swh-core (2.13.1-1~swh1~bpo10+1) buster-swh; urgency=medium +swh-core (2.14.0-1~swh1) unstable-swh; urgency=medium - * Rebuild for buster-swh + * New upstream release 2.14.0 - (tagged by Valentin Lorentz + on 2022-08-16 13:49:11 +0200) + * Upstream changes: - v2.14.0 - * RPC server: Do not log + exceptions with 4xx HTTP status codes - * Remove support for + deprecated exception format - * Make the RPC client raise a + specific exception class on 503 - -- Software Heritage autobuilder (on jenkins-debian1) Thu, 04 Aug 2022 08:47:14 +0000 + -- Software Heritage autobuilder (on jenkins-debian1) Tue, 16 Aug 2022 11:52:17 +0000 swh-core (2.13.1-1~swh1) unstable-swh; urgency=medium * New upstream release 2.13.1 - (tagged by Valentin Lorentz on 2022-08-04 10:42:39 +0200) * Upstream changes: - v2.13.1 - * Remove use of app.test_client() as a ctx-manager, to fix support - for Flask 2.2.0 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 04 Aug 2022 08:45:44 +0000 swh-core (2.13-1~swh1) unstable-swh; urgency=medium * New upstream release 2.13 - (tagged by David Douard on 2022-07-05 14:22:04 +0200) * Upstream changes: - v2.13 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 05 Jul 2022 12:38:24 +0000 swh-core (2.12-1~swh2) unstable-swh; urgency=medium * Bump new release with correct deps. -- Antoine R. Dumont (@ardumont) Mon, 20 Jun 2022 10:13:14 +0200 swh-core (2.12-1~swh1) unstable-swh; urgency=medium * New upstream release 2.12 - (tagged by Nicolas Dandrimont on 2022-06-16 13:38:52 +0200) * Upstream changes: - Release swh.core v2.12 - Add support for zstandard compressed tarballs -- Software Heritage autobuilder (on jenkins-debian1) Thu, 16 Jun 2022 11:42:04 +0000 swh-core (2.11-1~swh1) unstable-swh; urgency=medium * New upstream release 2.11 - (tagged by Antoine R. Dumont (@ardumont) on 2022-06-10 10:20:37 +0200) * Upstream changes: - v2.11 - tarball: Use standard Python module zipfile to extract jar archive - Add missing coverage on `swh db version` cli - db: Grant read access to guest user on all schema tables -- Software Heritage autobuilder (on jenkins-debian1) Fri, 10 Jun 2022 08:24:24 +0000 swh-core (2.10-1~swh1) unstable-swh; urgency=medium * New upstream release 2.10 - (tagged by Antoine R. Dumont (@ardumont) on 2022-06-02 16:03:41 +0200) * Upstream changes: - v2.10 - github/utils: Deal with exotic urls to canonicalize - deprecate the db/pytest_plugin.py module - move initialize_database_for_module in db_utils - mark postgresql_fact fixture factory function as deprecated - tests: use stock pytest_postgresql factory function - docs/db: Update datastore requirement -- Software Heritage autobuilder (on jenkins-debian1) Thu, 02 Jun 2022 14:39:11 +0000 swh-core (2.9.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.9.0 - (tagged by Antoine R. Dumont (@ardumont) on 2022-05-30 17:36:23 +0200) * Upstream changes: - v2.9.0 - Allow module to specify another config key than their module name - cli.db: Reword ignore sentence -- Software Heritage autobuilder (on jenkins-debian1) Mon, 30 May 2022 15:41:42 +0000 swh-core (2.8.1-1~swh1) unstable-swh; urgency=medium * New upstream release 2.8.1 - (tagged by Antoine R. Dumont (@ardumont) on 2022-05-30 17:06:11 +0200) * Upstream changes: - v2.8.1 - cli.db: Use attribute current_version instead of undeclared getter - cli.db: Fix help message typo -- Software Heritage autobuilder (on jenkins-debian1) Mon, 30 May 2022 15:10:17 +0000 swh-core (2.8.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.8.0 - (tagged by Antoine R. Dumont (@ardumont) on 2022-05-20 18:17:20 +0200) * Upstream changes: - v2.8.0 - Deal with git protocol url to canonicalize to https -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 May 2022 16:21:13 +0000 swh-core (2.7.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.7.0 - (tagged by Antoine R. Dumont (@ardumont) on 2022-05-20 15:26:30 +0200) * Upstream changes: - v2.7.0 - Use GitHubSession to make canonical computation deal with rate limit -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 May 2022 13:30:15 +0000 swh-core (2.6.0-1~swh2) unstable-swh; urgency=medium * Bump new release -- Antoine R. Dumont (@ardumont) Fri, 20 May 2022 10:26:03 +0200 swh-core (2.6.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.6.0 - (tagged by Antoine R. Dumont (@ardumont) on 2022-05-20 08:54:46 +0200) * Upstream changes: - v2.6.0 - Extract reusable github tests fixtures into its own pytest_plugin - test_db/test_db_copy_to: Fix hypothesis FailedHealthCheck error - Refactor swh.lister.github.utils to swh.core.github.utils - Add utility function to retrieve canonical github urls - Make 'python -m swh' work as cli entry point - db_utils: Make connect_to_conninfo use through contextmanager - Upgrade mypy to 0.942 to fix support of types-psycopg2 >= 2.9.12 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 May 2022 06:58:23 +0000 swh-core (2.5.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.5.0 - (tagged by Valentin Lorentz on 2022-04-25 13:51:26 +0200) * Upstream changes: - v2.5.0 - * Make db_transaction's client_options configurable at run time - * sentry: always override init settings with the environment variables - * Add support for disabling logging-based events in sentry - * RPC server: explicitly handle sentry exception capture (instead of gunicorn) - * statsd: add an error_type tag to @timed error counters - * cli: Ensure tests don't mess with the global logging setup - * mypy/pre-commit/pytest maintenance -- Software Heritage autobuilder (on jenkins-debian1) Mon, 25 Apr 2022 11:56:02 +0000 swh-core (2.4.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.4.0 - (tagged by Valentin Lorentz on 2022-03-29 14:39:15 +0200) * Upstream changes: - v2.4.0 - * Fix support of Werkzeug 2.1.0 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 29 Mar 2022 12:42:28 +0000 swh-core (2.3.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.3.0 - (tagged by David Douard on 2022-03-14 17:19:37 +0100) * Upstream changes: - v2.3.0 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 14 Mar 2022 16:23:52 +0000 swh-core (2.2.2-1~swh1) unstable-swh; urgency=medium * New upstream release 2.2.2 - (tagged by David Douard on 2022-03-09 17:44:25 +0100) * Upstream changes: - v2.2.2 - small fixes in the `swh db upgrade` command -- Software Heritage autobuilder (on jenkins-debian1) Wed, 09 Mar 2022 16:48:21 +0000 swh-core (2.2.1-1~swh1) unstable-swh; urgency=medium * New upstream release 2.2.1 - (tagged by Nicolas Dandrimont on 2022-03-03 20:20:11 +0100) * Upstream changes: - Release swh.core v2.2.1 - quiesce deprecation warnings for autogenerated RPC clients - fix typing for a few methods in db_utils -- Software Heritage autobuilder (on jenkins-debian1) Thu, 03 Mar 2022 19:24:59 +0000 swh-core (2.2.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.2.0 - (tagged by Valentin Lorentz on 2022-03-02 11:45:18 +0100) * Upstream changes: - v2.2.0 - * utils: Add a new 'iter_chunks' function -- Software Heritage autobuilder (on jenkins-debian1) Wed, 02 Mar 2022 10:47:47 +0000 swh-core (2.1.1-1~swh1) unstable-swh; urgency=medium * New upstream release 2.1.1 - (tagged by Valentin Lorentz on 2022-03-01 17:00:36 +0100) * Upstream changes: - v2.1.1 - * Fix wrong version numbers in deprecation message. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 01 Mar 2022 16:03:05 +0000 swh-core (2.1.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.1.0 - (tagged by Valentin Lorentz on 2022-03-01 16:58:02 +0100) * Upstream changes: - v2.1.0 - * RPCClient: Make methods {get,post}{,_stream} protected - * RPCServerApp: Add hooks to add behaviors to generated methods -- Software Heritage autobuilder (on jenkins-debian1) Tue, 01 Mar 2022 16:00:51 +0000 swh-core (2.0.0-2~swh1) unstable-swh; urgency=medium * Bump dependency constranints for pytest and pytest-postgresql. -- David Douard Wed, 23 Feb 2022 10:56:37 +0100 swh-core (2.0.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.0.0 - (tagged by David Douard on 2022-02-17 15:26:59 +0100) * Upstream changes: - v2.0.0 - add support for generic db version handling for postgresql backends, - add support for generic db upgrade (for postgresql backends), - upgrade pytest- postgresql based tests scaffolding to use the - template-based db creation (instead of the truncate-based db reset). -- Software Heritage autobuilder (on jenkins-debian1) Wed, 23 Feb 2022 09:34:41 +0000 swh-core (1.1.1-1~swh1) unstable-swh; urgency=medium * New upstream release 1.1.1 - (tagged by Valentin Lorentz on 2022-02-04 12:52:06 +0100) * Upstream changes: - v1.1.1 - * Require pytest to be <7.0.0 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 04 Feb 2022 11:54:45 +0000 swh-core (1.1.0-1~swh1) unstable-swh; urgency=medium * New upstream release 1.1.0 - (tagged by David Douard on 2022-01-20 15:39:02 +0100) * Upstream changes: - v1.1.0 - add a Statsd.status_gauge() context manager - add support for env var substitution in STATS_TAGS - pin mypy version and clean reauirements a bit -- Software Heritage autobuilder (on jenkins-debian1) Thu, 20 Jan 2022 14:44:14 +0000 swh-core (1.0.0-2~swh1) unstable-swh; urgency=medium * Add missing B-D on python3-blinker for sentry-sdk -- Nicolas Dandrimont Thu, 02 Dec 2021 12:34:50 +0100 swh-core (1.0.0-1~swh1) unstable-swh; urgency=medium * New upstream release 1.0.0 - (tagged by David Douard on 2021-12-02 10:51:10 +0100) * Upstream changes: - v1.0.0 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 02 Dec 2021 10:00:19 +0000 swh-core (0.15.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.15.1 - (tagged by Valentin Lorentz on 2021-11-08 14:10:08 +0100) * Upstream changes: - v0.15.1 - * Require pytest-postgresql < 4.0 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 08 Nov 2021 13:12:36 +0000 swh-core (0.15.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.15.0 - (tagged by Antoine R. Dumont (@ardumont) on 2021-09-20 11:45:26 +0200) * Upstream changes: - v0.15.0 - tarball: Fallback to guess archive format from mimetype when no format detected -- Software Heritage autobuilder (on jenkins-debian1) Mon, 20 Sep 2021 09:48:28 +0000 swh-core (0.14.6-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.6 - (tagged by Antoine Lambert on 2021-09-16 10:48:35 +0200) * Upstream changes: - version 0.14.6 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 16 Sep 2021 08:51:58 +0000 swh-core (0.14.5-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.5 - (tagged by Valentin Lorentz on 2021-08-30 10:31:47 +0200) * Upstream changes: - v0.14.5 - * tarball: Add support for .tbz, .tbz2, and .jar -- Software Heritage autobuilder (on jenkins-debian1) Mon, 30 Aug 2021 08:34:03 +0000 swh-core (0.14.4-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.4 - (tagged by Valentin Lorentz on 2021-07-30 16:44:26 +0200) * Upstream changes: - v0.14.4 - * add stream_results_optional -- Software Heritage autobuilder (on jenkins-debian1) Fri, 30 Jul 2021 14:47:25 +0000 swh-core (0.14.3-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.3 - (tagged by Antoine Lambert on 2021-06-11 15:41:38 +0200) * Upstream changes: - version 0.14.3 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 11 Jun 2021 13:47:13 +0000 swh-core (0.14.2-2~swh1) unstable-swh; urgency=medium * Rebuild v0.14.2 after missing unzip dependency -- Antoine Lambert Thu, 10 Jun 2021 16:24:09 +0200 swh-core (0.14.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.2 - (tagged by Antoine Lambert on 2021-06-10 16:09:06 +0200) * Upstream changes: - version 0.14.2 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 10 Jun 2021 14:13:25 +0000 swh-core (0.14.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.1 - (tagged by Valentin Lorentz on 2021-05-06 15:33:56 +0200) * Upstream changes: - v0.14.1 - * Fix reserved name being used in pytest plugin -- Software Heritage autobuilder (on jenkins-debian1) Thu, 06 May 2021 13:38:24 +0000 swh-core (0.14.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.0 - (tagged by Valentin Lorentz on 2021-05-06 14:14:10 +0200) * Upstream changes: - v0.14.0 - Add support for pytest- postgresql 3.0.0 - For consistency, I renamed db_name to dbname everywhere, so this will - affect other SWH packages. -- Software Heritage autobuilder (on jenkins-debian1) Thu, 06 May 2021 12:17:31 +0000 swh-core (0.13.3-1~swh1) unstable-swh; urgency=medium * New upstream release 0.13.3 - (tagged by Antoine Lambert on 2021-05-06 13:54:51 +0200) * Upstream changes: - version 0.13.3 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 06 May 2021 11:59:49 +0000 swh-core (0.13.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.13.2 - (tagged by Valentin Lorentz on 2021-05-06 10:40:51 +0200) * Upstream changes: - v0.13.2 - * tarball: properly normalize perms for all extracted files - * requirements-db-pytestplugin: Don't install pytest-postgresql 3.0+ -- Software Heritage autobuilder (on jenkins-debian1) Thu, 06 May 2021 08:44:45 +0000 swh-core (0.13.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.13.1 - (tagged by Antoine Lambert on 2021-04-29 14:21:29 +0200) * Upstream changes: - version 0.13.1 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 29 Apr 2021 12:25:15 +0000 swh-core (0.13.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.13.0 - (tagged by Vincent SELLIER on 2021-04-06 19:01:37 +0200) * Upstream changes: - v0.13.0 - Support several backends on RPCServerApp -- Software Heritage autobuilder (on jenkins-debian1) Tue, 06 Apr 2021 17:07:10 +0000 swh-core (0.12.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.12.1 - (tagged by Valentin Lorentz on 2021-04-06 12:34:34 +0200) * Upstream changes: - v0.12.1 - * tests: Drop hypothesis < 6 requirement - * README.rst: Remove getting-started instructions, they are duplicates - * Improve/fix documentation of requests_mock_datadir - * Remove dependency on 'decorator' (fixes a regression in decorator 5.0.5) -- Software Heritage autobuilder (on jenkins-debian1) Tue, 06 Apr 2021 10:37:17 +0000 swh-core (0.12.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.12.0 - (tagged by David Douard on 2021-02-16 11:48:58 +0100) * Upstream changes: - v0.12.0 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 16 Feb 2021 10:52:14 +0000 swh-core (0.11.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.11.0 - (tagged by David Douard on 2020-12-08 15:35:05 +0100) * Upstream changes: - v0.11.0 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 08 Dec 2020 14:38:23 +0000 swh-core (0.10.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.10.0 - (tagged by Nicolas Dandrimont on 2020-12-02 11:46:38 +0100) * Upstream changes: - Release swh.core 0.10.0 - db.tests.db_testing: Drop unused database test utilities. - api.serializers: Add support for serializing large negative integers with - msgpack. - Do not mutate api.serializers.ENCODERS or DECODERS. - swh cli: Add support for setting up the log level of multiple loggers. -- Software Heritage autobuilder (on jenkins-debian1) Wed, 02 Dec 2020 10:50:35 +0000 swh-core (0.9.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.9.1 - (tagged by Antoine Lambert on 2020-11-23 11:24:50 +0100) * Upstream changes: - version 0.9.1 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 23 Nov 2020 10:29:15 +0000 swh-core (0.9.0-1~swh2) unstable-swh; urgency=medium * Split packages python3-swh.core and python3-swh.core.db.pytestplugin -- Antoine R. Dumont (@ardumont) Fri, 20 Nov 2020 16:37:00 +0000 swh-core (0.9.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.9.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-11-20 15:35:53 +0100) * Upstream changes: - v0.9.0 - Clarify names around the swh.core.db.pytest_plugin tests - setup: Separate pytest- postgresql dependency and declare it when needed - RPCClient: Fix reraise_exceptions regression - api/serializers: Add Exception type encoder and decoder - Makefile.local: Ensure all tests are executed when invoking make test - core.db.cli: Add coverage and ensure `swh db *` works as expected - core tests: disambiguate arg 'request' through typing -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 Nov 2020 14:37:00 +0000 swh-core (0.8.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.8.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-30 09:12:17 +0100) * Upstream changes: - v0.8.0 - cli.db: Open init-admin subcmd to initialize superuser-level scripts -- Software Heritage autobuilder (on jenkins-debian1) Fri, 30 Oct 2020 08:13:02 +0000 swh-core (0.7.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.7.1 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-22 18:51:16 +0200) * Upstream changes: - v0.7.1 - Move SWHDatabaseJanitor to db.pytest_plugin module -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Oct 2020 16:51:59 +0000 swh-core (0.7.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.7.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-22 18:31:18 +0200) * Upstream changes: - v0.7.0 - remote_api_endpoint: Allow to declare what http method to use - api.RPCServerApp: Adapt sync rpc server with async rpc server -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Oct 2020 16:32:04 +0000 swh-core (0.6.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.6.1 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-22 13:53:41 +0200) * Upstream changes: - v0.6.1 - Move pytest_plugin declaration to top-level conftest -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Oct 2020 11:54:49 +0000 swh-core (0.6.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.6.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-22 13:24:55 +0200) * Upstream changes: - v0.6.0 - asynchronous.RPCServerApp: Align implementation with api.RPCServerApp -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Oct 2020 11:25:29 +0000 swh-core (0.5.0-1~swh2) unstable-swh; urgency=medium * Bump new release with dependency updated -- Antoine R. Dumont thu, 22 Oct 2020 11:21:04 +0200 swh-core (0.5.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.5.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-22 11:11:29 +0200) * Upstream changes: - v0.5.0 - Install postgresql_fact fixture for faster postgres tests - api.tests.test_async: Simplify fixture setup - core.config: Drop no longer used SWHConfig -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Oct 2020 09:12:14 +0000 swh-core (0.4.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.4.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-02 11:44:10 +0200) * Upstream changes: - v0.4.0 - config: Deprecate SWHConfig in favor of load_from_envvar function -- Software Heritage autobuilder (on jenkins-debian1) Fri, 02 Oct 2020 09:44:53 +0000 swh-core (0.3.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.1 - (tagged by Valentin Lorentz on 2020-10-01 12:38:08 +0200) * Upstream changes: - v0.3.1 - * Add specific celery task arguments to metadata sent to systemd-journald - * SortedList: Don't inherit from UserList. -- Software Heritage autobuilder (on jenkins-debian1) Thu, 01 Oct 2020 10:40:27 +0000 swh-core (0.3.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.0 - (tagged by David Douard on 2020-09-23 16:24:40 +0200) * Upstream changes: - v0.3.0 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 23 Sep 2020 14:27:08 +0000 swh-core (0.2.3-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.3 - (tagged by Valentin Lorentz on 2020-08-17 13:55:41 +0200) * Upstream changes: - v0.2.3 - * tarball: add test for permissions. - * Move SortedList from swh-storage. -- Software Heritage autobuilder (on jenkins-debian1) Mon, 17 Aug 2020 11:57:43 +0000 swh-core (0.2.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.2 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-31 13:41:19 +0200) * Upstream changes: - v0.2.2 - api.classes: Open swh.core.api.classes.stream_results -- Software Heritage autobuilder (on jenkins-debian1) Fri, 31 Jul 2020 11:44:33 +0000 swh-core (0.2.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.1 - (tagged by Valentin Lorentz on 2020-07-30 19:16:57 +0200) * Upstream changes: - v0.2.1 - Make @remote_api_endpoint preserve typing information for mypy. -- Software Heritage autobuilder (on jenkins-debian1) Thu, 30 Jul 2020 17:20:05 +0000 swh-core (0.2.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-29 11:18:37 +0200) * Upstream changes: - v0.2.0 - core.api: Expose serializable PagedResult object for pagination api - test_serializers: Refactor using pytest - Migrate from vcversioner to setuptools- scm -- Software Heritage autobuilder (on jenkins-debian1) Wed, 29 Jul 2020 09:20:56 +0000 swh-core (0.1.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.1.2 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-08 13:41:51 +0200) * Upstream changes: - v0.1.2 - test_serializers: Move to pytest for that specific erratic assertion -- Software Heritage autobuilder (on jenkins-debian1) Wed, 08 Jul 2020 11:43:25 +0000 swh-core (0.1.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.1.1 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-08 13:03:52 +0200) * Upstream changes: - v0.1.1 - api.tests: Fix unsupported matches keyword to match - requirements-db: Move typing- extension from test to runtime deps -- Software Heritage autobuilder (on jenkins-debian1) Wed, 08 Jul 2020 11:07:55 +0000 swh-core (0.1.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.1.0 - (tagged by David Douard on 2020-07-06 14:33:28 +0200) * Upstream changes: - v0.1.0 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 06 Jul 2020 12:36:15 +0000 swh-core (0.0.95-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.95 - (tagged by Nicolas Dandrimont on 2020-04-17 17:20:35 +0200) * Upstream changes: - Release swh.core v0.0.95 - support serializing large integers in msgpack - add documentation for CLI - move formatting to black -- Software Heritage autobuilder (on jenkins-debian1) Fri, 17 Apr 2020 16:05:01 +0000 swh-core (0.0.94-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.94 - (tagged by Valentin Lorentz on 2020-02-28 14:28:11 +0100) * Upstream changes: - v0.0.94 - Allow subclasses of RPCClient to override methods. -- Software Heritage autobuilder (on jenkins-debian1) Fri, 28 Feb 2020 13:32:41 +0000 swh-core (0.0.93-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.93 - (tagged by Valentin Lorentz on 2020-02-26 15:26:29 +0100) * Upstream changes: - v0.0.93 - Reintroduce support for decoding legacy msgpack encoding -- Software Heritage autobuilder (on jenkins-debian1) Wed, 26 Feb 2020 14:29:46 +0000 swh-core (0.0.92-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.92 - (tagged by Valentin Lorentz on 2020-02-19 15:40:57 +0100) * Upstream changes: - v0.0.92 - Add support for msgpack 1.0.0 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 19 Feb 2020 14:45:27 +0000 swh-core (0.0.91-1~swh2) unstable-swh; urgency=medium * Add missing python3-iso8601 build dependency -- Antoine Lambert Wed, 19 Feb 2020 11:18:34 +0100 swh-core (0.0.91-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.91 - (tagged by Antoine Lambert on 2020-02-18 16:43:59 +0100) * Upstream changes: - version 0.0.91 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 18 Feb 2020 15:51:39 +0000 swh-core (0.0.90-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.90 - (tagged by Valentin Lorentz on 2020-02-18 13:54:30 +0100) * Upstream changes: - v0.0.90 - Remove exception pickling from the async server, as in the sync server. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 18 Feb 2020 12:59:38 +0000 swh-core (0.0.89-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.89 - (tagged by Valentin Lorentz on 2020-02-18 11:35:01 +0100) * Upstream changes: - v0.0.89 - * Change msgpack serialization to be closer to the JSON one. - * Add support for extra {de,en}coders. - * Add extra_type_encoders and extra_type_decoders attributes to RPC clients and servers. - * Use iso8601.parse_date instead of dateutil.parser.parse. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 18 Feb 2020 10:43:34 +0000 swh-core (0.0.88-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.88 - (tagged by Valentin Lorentz on 2020-02-14 12:22:23 +0100) * Upstream changes: - v0.0.88 - In case of errors, return a simple dictionary instead of pickled exception. -- Software Heritage autobuilder (on jenkins-debian1) Fri, 14 Feb 2020 11:27:02 +0000 swh-core (0.0.87-1~swh2) unstable-swh; urgency=medium * Fix package build -- Antoine R. Dumont (@ardumont) Thu, 30 Jan 2020 14:06:54 +0100 swh-core (0.0.87-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.87 - (tagged by Valentin Lorentz on 2020-01-29 12:21:48 +0100) * Upstream changes: - v0.0.87 - Make db_transaction* remove db/cur from the signature. -- Software Heritage autobuilder (on jenkins-debian1) Wed, 29 Jan 2020 11:28:23 +0000 swh-core (0.0.86-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.86 - (tagged by Antoine R. Dumont (@ardumont) on 2020-01-23 09:08:56 +0100) * Upstream changes: - v0.0.86 - sentry: Add environment variable $SWH_SENTRY_ENVIRONMENT - pytest_plugin: Fix sphinx warnings -- Software Heritage autobuilder (on jenkins-debian1) Thu, 23 Jan 2020 08:12:40 +0000 swh-core (0.0.85-1~swh2) unstable-swh; urgency=medium * Fix package build on buster -- Antoine Lambert Thu, 16 Jan 2020 11:00:00 +0000 swh-core (0.0.85-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.85 - (tagged by Valentin Lorentz on 2020-01-15 13:07:36 +0100) * Upstream changes: - v0.0.85 - Add env var SWH_MAIN_PACKAGE -- Software Heritage autobuilder (on jenkins-debian1) Wed, 15 Jan 2020 12:11:49 +0000 swh-core (0.0.84-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.84 - (tagged by Antoine R. Dumont (@ardumont) on 2019-12-13 09:08:00 +0100) * Upstream changes: - v0.0.84 - Improve tarball support for tar.lz, tar.x, tar.Z files -- Software Heritage autobuilder (on jenkins-debian1) Fri, 13 Dec 2019 08:14:23 +0000 swh-core (0.0.83-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.83 - (tagged by Antoine R. Dumont (@ardumont) on 2019-12-12 16:03:25 +0100) * Upstream changes: - v0.0.83 - core.config: Rename configuration key -- Software Heritage autobuilder (on jenkins-debian1) Thu, 12 Dec 2019 15:06:28 +0000 swh-core (0.0.82-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.82 - (tagged by Nicolas Dandrimont on 2019-12-11 15:04:09 +0100) * Upstream changes: - Release swh.core 0.0.82 - Add missing conftest.py to MANIFEST.in -- Software Heritage autobuilder (on jenkins-debian1) Wed, 11 Dec 2019 14:09:00 +0000 swh-core (0.0.81-1~swh2) unstable-swh; urgency=medium * Add dependency to python3-sentry-sdk -- Nicolas Dandrimont Wed, 11 Dec 2019 14:58:54 +0100 swh-core (0.0.81-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.81 - (tagged by Valentin Lorentz on 2019-12-10 13:59:17 +0100) * Upstream changes: - v0.0.81 - * Include all requirements in MANIFEST.in - * Split test requirements to try and properly minimize dependencies - * Make the CLI initialize sentry-sdk based on CLI options/envvars. - * Add gunicorn config script to initialize sentry-sdk based on envvars. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 10 Dec 2019 13:03:13 +0000 swh-core (0.0.80-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.80 - (tagged by Nicolas Dandrimont on 2019-11-19 16:36:35 +0100) * Upstream changes: - Release swh.core v0.0.80 - Let TypeErrors pass through the RPC layer - Register SIGINT/SIGTERM handlers for the CLI -- Software Heritage autobuilder (on jenkins-debian1) Tue, 19 Nov 2019 15:41:09 +0000 swh-core (0.0.79-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.79 - (tagged by Stefano Zacchiroli on 2019-11-18 13:35:19 +0100) * Upstream changes: - v0.0.79 ======= - * RPCClient: add response attribute to RemoteException - * RPCClent: rename and refactor check_status (now raise_for_status) - * RPCClient: check HTTP status code for errors also when streaming - * cli: Add support for loading a logging configuration file - * cli: Allow adding a Notes section between Options and Commands - * Add trailing dot to help texts for consistency - * logger: only flatten dicts if all keys are strings - * Move to @pytest.fixture from yield_fixture - * test_rpc_client_server.py: fix typo in docstring -- Software Heritage autobuilder (on jenkins-debian1) Mon, 18 Nov 2019 12:42:12 +0000 swh-core (0.0.78-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.78 - (tagged by Nicolas Dandrimont on 2019-11-06 18:01:56 +0100) * Upstream changes: - Release swh.core 0.0.78 - allow the swh command to work even when a plugin fails - hardcode bytea and bytea[] type oids in BaseDb -- Software Heritage autobuilder (on jenkins-debian1) Wed, 06 Nov 2019 17:05:38 +0000 swh-core (0.0.77-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.77 - (tagged by Antoine R. Dumont (@ardumont) on 2019-11-06 14:10:58 +0100) * Upstream changes: - v0.0.77 - pytest_plugin: Decode url to resolve filename - api/serializers: Force json module use to decode requests text response -- Software Heritage autobuilder (on jenkins-debian1) Wed, 06 Nov 2019 13:15:03 +0000 swh-core (0.0.76-1~swh2) unstable-swh; urgency=medium * Force using the swh.core pytest plugin -- Nicolas Dandrimont Wed, 23 Oct 2019 14:50:04 +0200 swh-core (0.0.76-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.76 - (tagged by Nicolas Dandrimont on 2019-10-18 10:16:20 +0200) * Upstream changes: - Release swh.core v0.0.76 - Make the systemd dependency optional -- Software Heritage autobuilder (on jenkins-debian1) Fri, 18 Oct 2019 08:25:15 +0000 swh-core (0.0.75-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.75 - (tagged by Antoine R. Dumont (@ardumont) on 2019-10-14 17:51:58 +0200) * Upstream changes: - v0.0.75 - pytest_plugin: Add support for http requests -- Software Heritage autobuilder (on jenkins-debian1) Mon, 14 Oct 2019 15:57:05 +0000 swh-core (0.0.74-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.74 - (tagged by David Douard on 2019-10-11 15:30:51 +0200) * Upstream changes: - v0.0.74 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 11 Oct 2019 13:35:17 +0000 swh-core (0.0.73-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.73 - (tagged by Antoine R. Dumont (@ardumont) on 2019-10-09 14:16:04 +0200) * Upstream changes: - v0.0.73 - Improve pytest-plugin fixture to ease testing with pagination -- Software Heritage autobuilder (on jenkins-debian1) Wed, 09 Oct 2019 12:20:36 +0000 swh-core (0.0.72-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.72 - (tagged by Antoine R. Dumont (@ardumont) on 2019-10-09 10:59:28 +0200) * Upstream changes: - v0.0.72 - Fix tox.ini's py3 environment -- Software Heritage autobuilder (on jenkins-debian1) Wed, 09 Oct 2019 09:02:51 +0000 swh-core (0.0.70-1~swh2) unstable-swh; urgency=medium * Add new dependency on python3-tz -- Nicolas Dandrimont Tue, 01 Oct 2019 15:07:09 +0200 swh-core (0.0.70-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.70 - (tagged by Stefano Zacchiroli on 2019-09-27 10:16:41 +0200) * Upstream changes: - v0.0.70 - init.py: switch to documented way of extending path -- Software Heritage autobuilder (on jenkins-debian1) Fri, 27 Sep 2019 08:21:29 +0000 swh-core (0.0.69-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.69 - (tagged by Stefano Zacchiroli on 2019-09-20 15:50:52 +0200) * Upstream changes: - v0.0.69 - MANIFEST.in: ship py.typed -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 Sep 2019 13:54:15 +0000 swh-core (0.0.68-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.68 - (tagged by Stefano Zacchiroli on 2019-09-20 15:05:29 +0200) * Upstream changes: - v0.0.68 - * mypy: ignore django-stubs, needed only by hypothesis - * mypy: use conffile to ignore requests_mock - * typing: minimal changes to make a no-op mypy run pass - * db_testing.py: do not explode when TEST_DB_DUMP = None - * swh.core.config.parse_config_file: fix sphinx markup in docstring - * statsd: protect access to the statsd's socket - * tests: add tests for swh.logger and swh.tarball modules - * Remove fallback when aiohttp_utils is not installed. -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 Sep 2019 13:09:50 +0000 swh-core (0.0.67-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.67 - (tagged by Valentin Lorentz on 2019-08-22 13:56:36 +0200) * Upstream changes: - v0.0.67 - Improve error handling in Db.copy_to -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Aug 2019 12:02:14 +0000 swh-core (0.0.66-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.66 - (tagged by David Douard on 2019-07-30 13:55:16 +0200) * Upstream changes: - v0.0.66 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 30 Jul 2019 11:58:47 +0000 swh-core (0.0.65-1~swh2) unstable-swh; urgency=medium * debian/control: add missing dependencies. -- David Douard Tue, 16 Jul 2019 14:46:43 +0200 swh-core (0.0.65-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.65 - (tagged by David Douard on 2019-07-15 16:49:47 +0200) * Upstream changes: - v0.0.65 - needed to fix my mess with 0.0.64 tag, since the wrong 0.0.64 version has - already been pushed to pypi. -- Software Heritage autobuilder (on jenkins-debian1) Mon, 15 Jul 2019 14:53:29 +0000 swh-core (0.0.64-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.64 - (tagged by David Douard on 2019-07-15 16:33:32 +0200) * Upstream changes: - v0.0.64 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 15 Jul 2019 14:37:04 +0000 swh-core (0.0.63-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.63 - (tagged by Antoine Lambert on 2019-05-21 13:12:11 +0200) * Upstream changes: - version 0.0.63 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 21 May 2019 11:15:45 +0000 swh-core (0.0.62-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.62 - (tagged by Antoine Lambert on 2019-05-20 14:56:05 +0200) * Upstream changes: - version 0.0.62 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 20 May 2019 13:01:38 +0000 swh-core (0.0.61-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.61 - (tagged by David Douard on 2019-05-17 10:32:07 +0200) * Upstream changes: - v0.0.61 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 17 May 2019 08:38:08 +0000 swh-core (0.0.60-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.60 - (tagged by David Douard on 2019-05-06 15:27:44 +0200) * Upstream changes: - v0.0.60 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 06 May 2019 13:32:48 +0000 swh-core (0.0.59-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.59 - (tagged by Valentin Lorentz on 2019-04-09 16:55:41 +0200) * Upstream changes: - Explicitly give Db connections back to the pool. - So they gracefully release the connection on error instead - of relying on reference-counting to call the Db's `__del__` - (which does not happen in Hypothesis tests) because a ref - to it is kept via the traceback object. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 09 Apr 2019 16:12:32 +0000 swh-core (0.0.58-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.58 - (tagged by Antoine Lambert on 2019-04-02 17:19:05 +0200) * Upstream changes: - version 0.0.58 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 02 Apr 2019 15:24:34 +0000 swh-core (0.0.57-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.57 - (tagged by Nicolas Dandrimont on 2019-03-28 15:51:27 +0100) * Upstream changes: - Release swh.core v0.0.57 - Move to native async primitives - Fix statsd.timed exceptional behavior bug/misfeature - Fix SWHRemoteAPI post_stream method -- Software Heritage autobuilder (on jenkins-debian1) Thu, 28 Mar 2019 14:55:58 +0000 swh-core (0.0.56-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.56 - (tagged by David Douard on 2019-03-19 10:17:06 +0100) * Upstream changes: - v0.0.56 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 19 Mar 2019 09:27:18 +0000 swh-core (0.0.55-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.55 - (tagged by Antoine R. Dumont (@ardumont) on 2019-02-19 12:28:26 +0100) * Upstream changes: - v0.0.55 - Fix runtime dependencies -- Software Heritage autobuilder (on jenkins-debian1) Tue, 19 Feb 2019 11:32:28 +0000 swh-core (0.0.54-1~swh2) unstable-swh; urgency=medium * New upstream release 0.0.54 * Upstream changes: - Add missing build dependencies -- Antoine R. Dumont (@ardumont) Tue, 12 Feb 2019 16:25:34 +0000 swh-core (0.0.54-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.54 - (tagged by Valentin Lorentz on 2019-02-11 16:47:18 +0100) * Upstream changes: - Add test for BaseDb.connect. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 12 Feb 2019 12:37:43 +0000 swh-core (0.0.53-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.53 - (tagged by Antoine R. Dumont (@ardumont) on 2019-02-08 09:09:30 +0100) * Upstream changes: - v0.0.53 - Fix debian build -- Software Heritage autobuilder (on jenkins-debian1) Fri, 08 Feb 2019 08:12:31 +0000 swh-core (0.0.52-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.52 - (tagged by David Douard on 2019-02-06 15:24:04 +0100) * Upstream changes: - v0.0.52 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 06 Feb 2019 14:27:14 +0000 swh-core (0.0.51-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.51 - (tagged by David Douard on 2019-02-01 14:28:27 +0100) * Upstream changes: - v0.0.51 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 01 Feb 2019 13:31:45 +0000 swh-core (0.0.50-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.50 - (tagged by Nicolas Dandrimont on 2019-01-09 15:50:58 +0100) * Upstream changes: - Release swh.core v0.0.50 - Add statsd client module - Log used config files -- Software Heritage autobuilder (on jenkins-debian1) Wed, 09 Jan 2019 14:54:37 +0000 swh-core (0.0.49-1~swh1) unstable-swh; urgency=medium * Make DbTestFixture.setUp() accept and pass *args and **kwargs. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 08 Jan 2019 16:38:02 +0000 swh-core (0.0.48-1~swh1) unstable-swh; urgency=medium * v0.0.48 * swh.core.cli: Update swh-db-init to make it idemtpotent -- Antoine R. Dumont (@ardumont) Tue, 08 Jan 2019 15:33:15 +0000 swh-core (0.0.47-1~swh1) unstable-swh; urgency=medium * v0.0.47 * swh.core.cli: Fix flag -- Antoine R. Dumont (@ardumont) Tue, 08 Jan 2019 15:16:09 +0000 swh-core (0.0.46-1~swh1) unstable-swh; urgency=medium * v0.0.46 * utils.grouper: Improve implementation * Remove now-obsolete information about swh.core.worker -- Antoine R. Dumont (@ardumont) Tue, 08 Jan 2019 14:37:34 +0000 swh-core (0.0.45-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.45 * Compatibility with recent msgpack * Debian packaging-related cleanups -- Nicolas Dandrimont Thu, 22 Nov 2018 21:09:53 +0100 swh-core (0.0.44-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.44 * Refactor the database testing fixtures * Stop unsafe serialization/deserialization constructs * Update tests to use nose -- Nicolas Dandrimont Thu, 18 Oct 2018 18:20:12 +0200 swh-core (0.0.43-1~swh1) unstable-swh; urgency=medium * v0.0.43 * Fix missing dependency declaration -- Antoine R. Dumont (@ardumont) Thu, 11 Oct 2018 15:47:06 +0200 swh-core (0.0.42-1~swh1) unstable-swh; urgency=medium * v0.0.42 * Fix missing dependency declaration -- Antoine R. Dumont (@ardumont) Thu, 11 Oct 2018 15:45:25 +0200 swh-core (0.0.41-1~swh1) unstable-swh; urgency=medium * Add functions to generate HTTP API clients and servers from databases. * Summary: This moves the interesting parts of D505 into the core, so other components can use them as well. * Test Plan: `make test` * Reviewers: ardumont, seirl, #reviewers * Reviewed By: ardumont, #reviewers * Subscribers: douardda * Differential Revision: https://forge.softwareheritage.org/D507 -- Valentin Lorentz Thu, 11 Oct 2018 10:57:27 +0200 swh-core (0.0.40-1~swh1) unstable-swh; urgency=medium * v0.0.40 * swh.core.api.SWHRemoteAPI: Permit to set a query timeout option -- Antoine R. Dumont (@ardumont) Thu, 24 May 2018 12:10:03 +0200 swh-core (0.0.39-1~swh1) unstable-swh; urgency=medium * v0.0.39 * package: Add missing runtime dependency -- Antoine R. Dumont (@ardumont) Thu, 26 Apr 2018 15:24:22 +0200 swh-core (0.0.38-1~swh1) unstable-swh; urgency=medium * v0.0.38 * tests: Use more reasonable psql options for db restores * swh.core.serializers: Add custom types serialization -- Antoine R. Dumont (@ardumont) Thu, 26 Apr 2018 15:15:27 +0200 swh-core (0.0.37-1~swh1) unstable-swh; urgency=medium * v0.0.37 * Move test fixture in swh.core.tests.server_testing module -- Antoine R. Dumont (@ardumont) Wed, 25 Apr 2018 15:00:02 +0200 swh-core (0.0.36-1~swh1) unstable-swh; urgency=medium * v0.0.36 * Migrate swh.loader.tar.tarball module in swh.core -- Antoine R. Dumont (@ardumont) Wed, 06 Dec 2017 12:03:29 +0100 swh-core (0.0.35-1~swh1) unstable-swh; urgency=medium * Release swh.core version 0.0.35 * Update packaging runes -- Nicolas Dandrimont Thu, 12 Oct 2017 18:07:50 +0200 swh-core (0.0.34-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.34 * New modular database test fixture -- Nicolas Dandrimont Mon, 07 Aug 2017 18:29:48 +0200 swh-core (0.0.33-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.33 * Be more conservative with remote API responses -- Nicolas Dandrimont Mon, 19 Jun 2017 19:01:38 +0200 swh-core (0.0.32-1~swh1) unstable-swh; urgency=medium * Release swh-core v0.0.32 * Add asynchronous streaming methods for internal APIs * Remove task arguments from systemd-journal loggers -- Nicolas Dandrimont Tue, 09 May 2017 14:04:22 +0200 swh-core (0.0.31-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.31 * Add explicit dependency on python3-systemd -- Nicolas Dandrimont Fri, 07 Apr 2017 15:11:26 +0200 swh-core (0.0.30-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.30 * drop swh.core.hashutil (moved to swh.model.hashutil) * add a systemd logger -- Nicolas Dandrimont Fri, 07 Apr 2017 11:49:15 +0200 swh-core (0.0.29-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.29 * Catch proper exception in the base API client -- Nicolas Dandrimont Thu, 02 Feb 2017 00:19:25 +0100 swh-core (0.0.28-1~swh1) unstable-swh; urgency=medium * v0.0.28 * Refactoring some common code into swh.core -- Antoine R. Dumont (@ardumont) Thu, 26 Jan 2017 14:54:22 +0100 swh-core (0.0.27-1~swh1) unstable-swh; urgency=medium * v0.0.27 * Fix issue with default boolean value -- Antoine R. Dumont (@ardumont) Thu, 20 Oct 2016 16:15:20 +0200 swh-core (0.0.26-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.26 * Raise an exception when a configuration file exists and is unreadable -- Nicolas Dandrimont Wed, 12 Oct 2016 10:16:09 +0200 swh-core (0.0.25-1~swh1) unstable-swh; urgency=medium * v0.0.25 * Add new function utils.cwd -- Antoine R. Dumont (@ardumont) Thu, 29 Sep 2016 21:29:37 +0200 swh-core (0.0.24-1~swh1) unstable-swh; urgency=medium * v0.0.24 * Deal with edge case in logger regarding json -- Antoine R. Dumont (@ardumont) Thu, 22 Sep 2016 12:21:09 +0200 swh-core (0.0.23-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.23 * Properly fix the PyYAML dependency -- Nicolas Dandrimont Tue, 23 Aug 2016 16:20:29 +0200 swh-core (0.0.22-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.22 * Proper loading of yaml and ini files in all paths -- Nicolas Dandrimont Fri, 19 Aug 2016 15:45:55 +0200 swh-core (0.0.21-1~swh1) unstable-swh; urgency=medium * v0.0.21 * Update test tools -- Antoine R. Dumont (@ardumont) Tue, 19 Jul 2016 14:47:01 +0200 swh-core (0.0.20-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.20 * Add some generic bytes <-> escaped unicode methods -- Nicolas Dandrimont Tue, 14 Jun 2016 16:54:41 +0200 swh-core (0.0.19-1~swh1) unstable-swh; urgency=medium * v0.0.19 * Resurrect swh.core.utils -- Antoine R. Dumont (@ardumont) Fri, 15 Apr 2016 12:40:43 +0200 swh-core (0.0.18-1~swh1) unstable-swh; urgency=medium * v0.0.18 * Add swh.core.utils * serializers: support UUIDs all around -- Antoine R. Dumont (@ardumont) Sat, 26 Mar 2016 11:16:33 +0100 swh-core (0.0.17-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.17 * Allow serialization of UUIDs -- Nicolas Dandrimont Fri, 04 Mar 2016 11:40:56 +0100 swh-core (0.0.16-1~swh1) unstable-swh; urgency=medium * Release swh.core version 0.0.16 * add bytehex_to_hash and hash_to_bytehex in hashutil * move scheduling utilities to swh.scheduler -- Nicolas Dandrimont Fri, 19 Feb 2016 18:12:10 +0100 swh-core (0.0.15-1~swh1) unstable-swh; urgency=medium * Release v0.0.15 * Add hashutil.hash_git_object -- Nicolas Dandrimont Wed, 16 Dec 2015 16:31:26 +0100 swh-core (0.0.14-1~swh1) unstable-swh; urgency=medium * v0.0.14 * Add simple README * Update license * swh.core.hashutil.hashfile can now deal with filepath as bytes -- Antoine R. Dumont (@ardumont) Fri, 23 Oct 2015 11:13:14 +0200 swh-core (0.0.13-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.13 -- Nicolas Dandrimont Fri, 09 Oct 2015 17:32:49 +0200 swh-core (0.0.12-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.12 -- Nicolas Dandrimont Tue, 06 Oct 2015 17:34:34 +0200 swh-core (0.0.11-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.11 -- Nicolas Dandrimont Sat, 03 Oct 2015 15:57:03 +0200 swh-core (0.0.10-1~swh1) unstable-swh; urgency=medium * Prepare deploying swh.core v0.0.10 -- Nicolas Dandrimont Sat, 03 Oct 2015 12:28:52 +0200 swh-core (0.0.9-1~swh1) unstable-swh; urgency=medium * Prepare deploying swh.core v0.0.9 -- Nicolas Dandrimont Sat, 03 Oct 2015 11:36:55 +0200 swh-core (0.0.8-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.8 -- Nicolas Dandrimont Thu, 01 Oct 2015 12:31:44 +0200 swh-core (0.0.7-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.7 -- Nicolas Dandrimont Thu, 01 Oct 2015 11:29:04 +0200 swh-core (0.0.6-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.6 -- Nicolas Dandrimont Tue, 29 Sep 2015 16:48:44 +0200 swh-core (0.0.5-1~swh1) unstable-swh; urgency=medium * Prepare v0.0.5 deployment -- Nicolas Dandrimont Tue, 29 Sep 2015 16:08:32 +0200 swh-core (0.0.4-1~swh1) unstable-swh; urgency=medium * Tagging swh.core 0.0.4 -- Nicolas Dandrimont Fri, 25 Sep 2015 15:41:26 +0200 swh-core (0.0.3-1~swh1) unstable-swh; urgency=medium * Tag swh.core v0.0.3 -- Nicolas Dandrimont Fri, 25 Sep 2015 11:07:10 +0200 swh-core (0.0.2-1~swh1) unstable-swh; urgency=medium * Deploy v0.0.2 -- Nicolas Dandrimont Wed, 23 Sep 2015 12:08:50 +0200 swh-core (0.0.1-1~swh1) unstable-swh; urgency=medium * Initial release * Tag v0.0.1 for deployment -- Nicolas Dandrimont Tue, 22 Sep 2015 14:52:26 +0200 diff --git a/swh.core.egg-info/PKG-INFO b/swh.core.egg-info/PKG-INFO index 6e7de3a..f08d69f 100644 --- a/swh.core.egg-info/PKG-INFO +++ b/swh.core.egg-info/PKG-INFO @@ -1,39 +1,39 @@ Metadata-Version: 2.1 Name: swh.core -Version: 2.13.1 +Version: 2.14.0 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-core Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-core/ Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/x-rst Provides-Extra: testing-core Provides-Extra: logging Provides-Extra: db Provides-Extra: http Provides-Extra: github Provides-Extra: testing License-File: LICENSE License-File: AUTHORS Software Heritage - Core foundations ==================================== Low-level utilities and helpers used by almost all other modules in the stack. core library for swh's modules: - config parser - serialization - logging mechanism - database connection - http-based RPC client/server diff --git a/swh/core/api/__init__.py b/swh/core/api/__init__.py index fa6977f..90b5793 100644 --- a/swh/core/api/__init__.py +++ b/swh/core/api/__init__.py @@ -1,520 +1,514 @@ -# Copyright (C) 2015-2020 The Software Heritage developers +# Copyright (C) 2015-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from collections import abc import functools import inspect import logging -import pickle from typing import ( Any, Callable, ClassVar, Dict, List, Optional, Tuple, Type, TypeVar, Union, ) from deprecated import deprecated from flask import Flask, Request, Response, abort, request import requests import sentry_sdk from werkzeug.exceptions import HTTPException from .negotiation import Formatter as FormatterBase from .negotiation import Negotiator as NegotiatorBase from .negotiation import negotiate as _negotiate from .serializers import ( exception_to_dict, json_dumps, json_loads, msgpack_dumps, msgpack_loads, ) from .serializers import decode_response from .serializers import encode_data_client as encode_data logger = logging.getLogger(__name__) # support for content negotiation class Negotiator(NegotiatorBase): def best_mimetype(self): return request.accept_mimetypes.best_match( self.accept_mimetypes, "application/json" ) def _abort(self, status_code, err=None): return abort(status_code, err) def negotiate(formatter_cls, *args, **kwargs): return _negotiate(Negotiator, formatter_cls, *args, **kwargs) class Formatter(FormatterBase): def _make_response(self, body, content_type): return Response(body, content_type=content_type) def configure(self, extra_encoders=None): self.extra_encoders = extra_encoders class JSONFormatter(Formatter): format = "json" mimetypes = ["application/json"] def render(self, obj): return json_dumps(obj, extra_encoders=self.extra_encoders) class MsgpackFormatter(Formatter): format = "msgpack" mimetypes = ["application/x-msgpack"] def render(self, obj): return msgpack_dumps(obj, extra_encoders=self.extra_encoders) # base API classes class RemoteException(Exception): """raised when remote returned an out-of-band failure notification, e.g., as a HTTP status code or serialized exception Attributes: response: HTTP response corresponding to the failure """ def __init__( self, payload: Optional[Any] = None, response: Optional[requests.Response] = None, ): if payload is not None: super().__init__(payload) else: super().__init__() self.response = response def __str__(self): if ( self.args and isinstance(self.args[0], dict) and "type" in self.args[0] and "args" in self.args[0] ): return ( f"' ) else: return super().__str__() +class TransientRemoteException(RemoteException): + """Subclass of RemoteException representing errors which are expected + to be temporary. + """ + + F = TypeVar("F", bound=Callable) def remote_api_endpoint(path: str, method: str = "POST") -> Callable[[F], F]: def dec(f: F) -> F: f._endpoint_path = path # type: ignore f._method = method # type: ignore return f return dec class APIError(Exception): """API Error""" def __str__(self): return "An unexpected error occurred in the backend: {}".format(self.args) class MetaRPCClient(type): """Metaclass for RPCClient, which adds a method for each endpoint of the database it is designed to access. See for example :class:`swh.indexer.storage.api.client.RemoteStorage`""" def __new__(cls, name, bases, attributes): # For each method wrapped with @remote_api_endpoint in an API backend # (eg. :class:`swh.indexer.storage.IndexerStorage`), add a new # method in RemoteStorage, with the same documentation. # # Note that, despite the usage of decorator magic (eg. functools.wrap), # this never actually calls an IndexerStorage method. backend_class = attributes.get("backend_class", None) for base in bases: if backend_class is not None: break backend_class = getattr(base, "backend_class", None) if backend_class: for (meth_name, meth) in backend_class.__dict__.items(): if hasattr(meth, "_endpoint_path"): cls.__add_endpoint(meth_name, meth, attributes) return super().__new__(cls, name, bases, attributes) @staticmethod def __add_endpoint(meth_name, meth, attributes): wrapped_meth = inspect.unwrap(meth) @functools.wraps(meth) # Copy signature and doc def meth_(*args, **kwargs): # Match arguments and parameters post_data = inspect.getcallargs(wrapped_meth, *args, **kwargs) # Remove arguments that should not be passed self = post_data.pop("self") post_data.pop("cur", None) post_data.pop("db", None) # Send the request. return self._post(meth._endpoint_path, post_data) if meth_name not in attributes: attributes[meth_name] = meth_ class RPCClient(metaclass=MetaRPCClient): """Proxy to an internal SWH RPC""" backend_class = None # type: ClassVar[Optional[type]] """For each method of `backend_class` decorated with :func:`remote_api_endpoint`, a method with the same prototype and docstring will be added to this class. Calls to this new method will be translated into HTTP requests to a remote server. This backend class will never be instantiated, it only serves as a template.""" api_exception = APIError # type: ClassVar[Type[Exception]] """The exception class to raise in case of communication error with the server.""" reraise_exceptions: ClassVar[List[Type[Exception]]] = [] """On server errors, if any of the exception classes in this list has the same name as the error name, then the exception will be instantiated and raised instead of a generic RemoteException.""" extra_type_encoders: List[Tuple[type, str, Callable]] = [] """Value of `extra_encoders` passed to `json_dumps` or `msgpack_dumps` to be able to serialize more object types.""" extra_type_decoders: Dict[str, Callable] = {} """Value of `extra_decoders` passed to `json_loads` or `msgpack_loads` to be able to deserialize more object types.""" def __init__( self, url, api_exception=None, timeout=None, chunk_size=4096, reraise_exceptions=None, **kwargs, ): if api_exception: self.api_exception = api_exception if reraise_exceptions: self.reraise_exceptions = reraise_exceptions base_url = url if url.endswith("/") else url + "/" self.url = base_url self.session = requests.Session() adapter = requests.adapters.HTTPAdapter( max_retries=kwargs.get("max_retries", 3), pool_connections=kwargs.get("pool_connections", 20), pool_maxsize=kwargs.get("pool_maxsize", 100), ) self.session.mount(self.url, adapter) self.timeout = timeout self.chunk_size = chunk_size def _url(self, endpoint): return "%s%s" % (self.url, endpoint) def raw_verb(self, verb, endpoint, **opts): if "chunk_size" in opts: # if the chunk_size argument has been passed, consider the user # also wants stream=True, otherwise, what's the point. opts["stream"] = True if self.timeout and "timeout" not in opts: opts["timeout"] = self.timeout try: return getattr(self.session, verb)(self._url(endpoint), **opts) except requests.exceptions.ConnectionError as e: raise self.api_exception(e) def _post(self, endpoint, data, **opts): if isinstance(data, (abc.Iterator, abc.Generator)): data = (self._encode_data(x) for x in data) else: data = self._encode_data(data) chunk_size = opts.pop("chunk_size", self.chunk_size) response = self.raw_verb( "post", endpoint, data=data, headers={ "content-type": "application/x-msgpack", "accept": "application/x-msgpack", }, **opts, ) if opts.get("stream") or response.headers.get("transfer-encoding") == "chunked": self.raise_for_status(response) return response.iter_content(chunk_size) else: return self._decode_response(response) def _encode_data(self, data): return encode_data(data, extra_encoders=self.extra_type_encoders) _post_stream = _post @deprecated(version="2.1.0", reason="Use _post instead") def post(self, *args, **kwargs): return self._post(*args, **kwargs) @deprecated(version="2.1.0", reason="Use _post_stream instead") def post_stream(self, *args, **kwargs): return self._post_stream(*args, **kwargs) def _get(self, endpoint, **opts): chunk_size = opts.pop("chunk_size", self.chunk_size) response = self.raw_verb( "get", endpoint, headers={"accept": "application/x-msgpack"}, **opts ) if opts.get("stream") or response.headers.get("transfer-encoding") == "chunked": self.raise_for_status(response) return response.iter_content(chunk_size) else: return self._decode_response(response) def _get_stream(self, endpoint, **opts): return self._get(endpoint, stream=True, **opts) @deprecated(version="2.1.0", reason="Use _get instead") def get(self, *args, **kwargs): return self._get(*args, **kwargs) @deprecated(version="2.1.0", reason="Use _get_stream instead") def get_stream(self, *args, **kwargs): return self._get_stream(*args, **kwargs) def raise_for_status(self, response) -> None: """check response HTTP status code and raise an exception if it denotes an error; do nothing otherwise """ status_code = response.status_code status_class = response.status_code // 100 if status_code == 404: raise RemoteException(payload="404 not found", response=response) exception = None - # TODO: only old servers send pickled error; stop trying to unpickle - # after they are all upgraded - try: - if status_class == 4: - data = self._decode_response(response, check_status=False) - if isinstance(data, dict): - # TODO: remove "exception" key check once all servers - # are using new schema - exc_data = data["exception"] if "exception" in data else data - for exc_type in self.reraise_exceptions: - if exc_type.__name__ == exc_data["type"]: - exception = exc_type(*exc_data["args"]) - break - else: - exception = RemoteException(payload=exc_data, response=response) - else: - exception = pickle.loads(data) - - elif status_class == 5: - data = self._decode_response(response, check_status=False) - if "exception_pickled" in data: - exception = pickle.loads(data["exception_pickled"]) - else: - # TODO: remove "exception" key check once all servers - # are using new schema - exc_data = data["exception"] if "exception" in data else data - exception = RemoteException(payload=exc_data, response=response) - - except (TypeError, pickle.UnpicklingError): - raise RemoteException(payload=data, response=response) + if status_class == 4: + exc_data = self._decode_response(response, check_status=False) + for exc_type in self.reraise_exceptions: + if exc_type.__name__ == exc_data["type"]: + exception = exc_type(*exc_data["args"]) + break + else: + exception = RemoteException(payload=exc_data, response=response) + + elif status_class == 5: + cls: Type[RemoteException] + if status_code == 503: + # This isn't a generic HTTP client and we know the server does + # not support the Retry-After header, so we do not implement + # it here either. + cls = TransientRemoteException + else: + cls = RemoteException + exc_data = self._decode_response(response, check_status=False) + exception = cls(payload=exc_data, response=response) if exception: raise exception from None if status_class != 2: raise RemoteException( payload=f"API HTTP error: {status_code} {response.content}", response=response, ) def _decode_response(self, response, check_status=True): if check_status: self.raise_for_status(response) return decode_response(response, extra_decoders=self.extra_type_decoders) def __repr__(self): return "<{} url={}>".format(self.__class__.__name__, self.url) class BytesRequest(Request): """Request with proper escaping of arbitrary byte sequences.""" encoding = "utf-8" encoding_errors = "surrogateescape" ENCODERS: Dict[str, Callable[[Any], Union[bytes, str]]] = { "application/x-msgpack": msgpack_dumps, "application/json": json_dumps, } def encode_data_server( data, content_type="application/x-msgpack", extra_type_encoders=None ): encoded_data = ENCODERS[content_type](data, extra_encoders=extra_type_encoders) return Response( encoded_data, mimetype=content_type, ) def decode_request(request, extra_decoders=None): content_type = request.mimetype data = request.get_data() if not data: return {} if content_type == "application/x-msgpack": r = msgpack_loads(data, extra_decoders=extra_decoders) elif content_type == "application/json": # XXX this .decode() is needed for py35. # Should not be needed any more with py37 r = json_loads(data.decode("utf-8"), extra_decoders=extra_decoders) else: raise ValueError("Wrong content type `%s` for API request" % content_type) return r def error_handler(exception, encoder, status_code=500): """Error handler to be registered using flask's error-handling decorator ``app.errorhandler``. This is used for exceptions that are expected in the normal execution flow of the RPC-ed API, in which case the status code should be set to a value in the 4xx range, as well as for exceptions that are unexpected (generally, a bare :class:`Exception`), and for which the status code should be kept in the 5xx class. This function only captures exceptions as sentry errors if the status code is in the 5xx range, as "expected exceptions" in the 4xx range are more likely to be handled on the client side. """ - logging.exception(exception) - status_class = status_code // 100 if status_class == 5: + logger.exception(exception) sentry_sdk.capture_exception(exception) response = encoder(exception_to_dict(exception)) if isinstance(exception, HTTPException): response.status_code = exception.code else: # TODO: differentiate between server errors and client errors response.status_code = status_code return response class RPCServerApp(Flask): """For each endpoint of the given `backend_class`, tells app.route to call a function that decodes the request and sends it to the backend object provided by the factory. :param Any backend_class: The class of the backend, which will be analyzed to look for API endpoints. :param Optional[Callable[[], backend_class]] backend_factory: A function with no argument that returns an instance of `backend_class`. If unset, defaults to calling `backend_class` constructor directly. For each method 'do_x()' of the ``backend_factory``, subclasses may implement two methods: ``pre_do_x(self, kw)`` and ``post_do_x(self, ret, kw)`` that will be called respectively before and after ``do_x(**kw)``. ``kw`` is the dict of request parameters, and ``ret`` is the return value of ``do_x(**kw)``. """ request_class = BytesRequest extra_type_encoders: List[Tuple[type, str, Callable]] = [] """Value of `extra_encoders` passed to `json_dumps` or `msgpack_dumps` to be able to serialize more object types.""" extra_type_decoders: Dict[str, Callable] = {} """Value of `extra_decoders` passed to `json_loads` or `msgpack_loads` to be able to deserialize more object types.""" method_decorators: List[Callable[[Callable], Callable]] = [] """List of decorators to all methods generated from the ``backend_class``.""" def __init__(self, *args, backend_class=None, backend_factory=None, **kwargs): super().__init__(*args, **kwargs) self.add_backend_class(backend_class, backend_factory) def add_backend_class(self, backend_class=None, backend_factory=None): if backend_class is None and backend_factory is not None: raise ValueError( "backend_factory should only be provided if backend_class is" ) if backend_class is not None: backend_factory = backend_factory or backend_class for (meth_name, meth) in backend_class.__dict__.items(): if hasattr(meth, "_endpoint_path"): self.__add_endpoint(meth_name, meth, backend_factory) def __add_endpoint(self, meth_name, meth, backend_factory): from flask import request @negotiate(MsgpackFormatter, extra_encoders=self.extra_type_encoders) @negotiate(JSONFormatter, extra_encoders=self.extra_type_encoders) @functools.wraps(meth) # Copy signature and doc def f(): # Call the actual code pre_hook = getattr(self, f"pre_{meth_name}", None) post_hook = getattr(self, f"post_{meth_name}", None) obj_meth = getattr(backend_factory(), meth_name) kw = decode_request(request, extra_decoders=self.extra_type_decoders) if pre_hook is not None: pre_hook(kw) ret = obj_meth(**kw) if post_hook is not None: post_hook(ret, kw) return ret for decorator in self.method_decorators: f = decorator(f) self.route("/" + meth._endpoint_path, methods=["POST"])(f) diff --git a/swh/core/api/asynchronous.py b/swh/core/api/asynchronous.py index 4e2188d..323d305 100644 --- a/swh/core/api/asynchronous.py +++ b/swh/core/api/asynchronous.py @@ -1,183 +1,185 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from collections import OrderedDict import functools import logging from typing import Callable, Dict, List, Optional, Tuple, Type, Union import aiohttp.web from aiohttp_utils import Response, negotiation from deprecated import deprecated import multidict from .serializers import ( exception_to_dict, json_dumps, json_loads, msgpack_dumps, msgpack_loads, ) +logger = logging.getLogger(__name__) + def encode_msgpack(data, **kwargs): return aiohttp.web.Response( body=msgpack_dumps(data), headers=multidict.MultiDict({"Content-Type": "application/x-msgpack"}), **kwargs, ) encode_data_server = Response def render_msgpack(request, data, extra_encoders=None): return msgpack_dumps(data, extra_encoders=extra_encoders) def render_json(request, data, extra_encoders=None): return json_dumps(data, extra_encoders=extra_encoders) def decode_data(data, content_type, extra_decoders=None): """Decode data according to content type, eventually using some extra decoders.""" if not data: return {} if content_type == "application/x-msgpack": r = msgpack_loads(data, extra_decoders=extra_decoders) elif content_type == "application/json": r = json_loads(data, extra_decoders=extra_decoders) else: raise ValueError(f"Wrong content type `{content_type}` for API request") return r async def decode_request(request, extra_decoders=None): """Decode asynchronously the request""" data = await request.read() return decode_data(data, request.content_type, extra_decoders=extra_decoders) async def error_middleware(app, handler): async def middleware_handler(request): try: return await handler(request) except Exception as e: if isinstance(e, aiohttp.web.HTTPException): raise - logging.exception(e) + logger.exception(e) res = exception_to_dict(e) if isinstance(e, app.client_exception_classes): status = 400 else: status = 500 return encode_data_server(res, status=status) return middleware_handler class RPCServerApp(aiohttp.web.Application): """For each endpoint of the given `backend_class`, tells app.route to call a function that decodes the request and sends it to the backend object provided by the factory. :param Any backend_class: The class of the backend, which will be analyzed to look for API endpoints. :param Optional[Callable[[], backend_class]] backend_factory: A function with no argument that returns an instance of `backend_class`. If unset, defaults to calling `backend_class` constructor directly. """ client_exception_classes: Tuple[Type[Exception], ...] = () """Exceptions that should be handled as a client error (eg. object not found, invalid argument)""" extra_type_encoders: List[Tuple[type, str, Callable]] = [] """Value of `extra_encoders` passed to `json_dumps` or `msgpack_dumps` to be able to serialize more object types.""" extra_type_decoders: Dict[str, Callable] = {} """Value of `extra_decoders` passed to `json_loads` or `msgpack_loads` to be able to deserialize more object types.""" def __init__( self, app_name: Optional[str] = None, backend_class: Optional[Callable] = None, backend_factory: Optional[Union[Callable, str]] = None, middlewares=(), **kwargs, ): nego_middleware = negotiation.negotiation_middleware( renderers=self._renderers(), force_rendering=True ) middlewares = ( nego_middleware, error_middleware, ) + middlewares super().__init__(middlewares=middlewares, **kwargs) # swh decorations starts here self.app_name = app_name if backend_class is None and backend_factory is not None: raise ValueError( "backend_factory should only be provided if backend_class is" ) self.backend_class = backend_class if backend_class is not None: backend_factory = backend_factory or backend_class for (meth_name, meth) in backend_class.__dict__.items(): if hasattr(meth, "_endpoint_path"): path = meth._endpoint_path http_method = meth._method path = path if path.startswith("/") else f"/{path}" self.router.add_route( http_method, path, self._endpoint(meth_name, meth, backend_factory), ) def _renderers(self): """Return an ordered list of renderers in order of increasing desirability (!) See mimetype.best_match() docstring """ return OrderedDict( [ ( "application/json", lambda request, data: render_json( request, data, extra_encoders=self.extra_type_encoders ), ), ( "application/x-msgpack", lambda request, data: render_msgpack( request, data, extra_encoders=self.extra_type_encoders ), ), ] ) def _endpoint(self, meth_name, meth, backend_factory): """Create endpoint out of the method `meth`.""" @functools.wraps(meth) # Copy signature and doc async def decorated_meth(request, *args, **kwargs): obj_meth = getattr(backend_factory(), meth_name) data = await request.read() kw = decode_data( data, request.content_type, extra_decoders=self.extra_type_decoders ) result = obj_meth(**kw) return encode_data_server(result) return decorated_meth @deprecated(version="0.0.64", reason="Use the RPCServerApp instead") class SWHRemoteAPI(RPCServerApp): pass diff --git a/swh/core/api/tests/test_rpc_client.py b/swh/core/api/tests/test_rpc_client.py index 6376f87..caa3f1c 100644 --- a/swh/core/api/tests/test_rpc_client.py +++ b/swh/core/api/tests/test_rpc_client.py @@ -1,182 +1,177 @@ -# Copyright (C) 2018-2019 The Software Heritage developers +# Copyright (C) 2018-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import re import pytest from requests.exceptions import ConnectionError -from swh.core.api import APIError, RemoteException, RPCClient, remote_api_endpoint +from swh.core.api import ( + APIError, + RemoteException, + RPCClient, + TransientRemoteException, + remote_api_endpoint, +) from swh.core.api.serializers import exception_to_dict, msgpack_dumps from .test_serializers import ExtraType, extra_decoders, extra_encoders class ReraiseException(Exception): pass @pytest.fixture def rpc_client(requests_mock): class TestStorage: @remote_api_endpoint("test_endpoint_url") def test_endpoint(self, test_data, db=None, cur=None): ... @remote_api_endpoint("path/to/endpoint") def something(self, data, db=None, cur=None): ... @remote_api_endpoint("serializer_test") def serializer_test(self, data, db=None, cur=None): ... @remote_api_endpoint("overridden/endpoint") def overridden_method(self, data): return "foo" class Testclient(RPCClient): backend_class = TestStorage extra_type_encoders = extra_encoders extra_type_decoders = extra_decoders reraise_exceptions = [ReraiseException] def overridden_method(self, data): return "bar" def callback(request, context): assert request.headers["Content-Type"] == "application/x-msgpack" context.headers["Content-Type"] = "application/x-msgpack" if request.path == "/test_endpoint_url": context.content = b"\xa3egg" elif request.path == "/path/to/endpoint": context.content = b"\xa4spam" elif request.path == "/serializer_test": context.content = ( b"\x82\xc4\x07swhtype\xa9extratype" b"\xc4\x01d\x92\x81\xa4spam\xa3egg\xa3qux" ) else: assert False return context.content requests_mock.post(re.compile("mock://example.com/"), content=callback) return Testclient(url="mock://example.com") def test_client(rpc_client): assert hasattr(rpc_client, "test_endpoint") assert hasattr(rpc_client, "something") res = rpc_client.test_endpoint("spam") assert res == "egg" res = rpc_client.test_endpoint(test_data="spam") assert res == "egg" res = rpc_client.something("whatever") assert res == "spam" res = rpc_client.something(data="whatever") assert res == "spam" def test_client_extra_serializers(rpc_client): res = rpc_client.serializer_test(["foo", ExtraType("bar", b"baz")]) assert res == ExtraType({"spam": "egg"}, "qux") def test_client_overridden_method(rpc_client): res = rpc_client.overridden_method("foo") assert res == "bar" def test_client_connexion_error(rpc_client, requests_mock): """ ConnectionError should be wrapped and raised as an APIError. """ error_message = "unreachable host" requests_mock.post( re.compile("mock://example.com/connection_error"), exc=ConnectionError(error_message), ) with pytest.raises(APIError) as exc_info: rpc_client._post("connection_error", data={}) assert type(exc_info.value.args[0]) == ConnectionError assert str(exc_info.value.args[0]) == error_message -def _exception_response(exception, status_code, old_exception_schema=False): +def _exception_response(exception, status_code): def callback(request, context): assert request.headers["Content-Type"] == "application/x-msgpack" context.headers["Content-Type"] = "application/x-msgpack" exc_dict = exception_to_dict(exception) - if old_exception_schema: - exc_dict = {"exception": exc_dict} context.content = msgpack_dumps(exc_dict) context.status_code = status_code return context.content return callback -@pytest.mark.parametrize("old_exception_schema", [False, True]) -def test_client_reraise_exception(rpc_client, requests_mock, old_exception_schema): +def test_client_reraise_exception(rpc_client, requests_mock): """ Exception caught server-side and whitelisted will be raised again client-side. """ error_message = "something went wrong" endpoint = "reraise_exception" requests_mock.post( re.compile(f"mock://example.com/{endpoint}"), content=_exception_response( exception=ReraiseException(error_message), status_code=400, - old_exception_schema=old_exception_schema, ), ) with pytest.raises(ReraiseException) as exc_info: rpc_client._post(endpoint, data={}) assert str(exc_info.value) == error_message -@pytest.mark.parametrize( - "status_code, old_exception_schema", - [ - (400, False), - (500, False), - (400, True), - (500, True), - ], -) -def test_client_raise_remote_exception( - rpc_client, requests_mock, status_code, old_exception_schema -): +@pytest.mark.parametrize("status_code", [400, 500, 503]) +def test_client_raise_remote_exception(rpc_client, requests_mock, status_code): """ Exception caught server-side and not whitelisted will be wrapped and raised as a RemoteException client-side. """ error_message = "something went wrong" endpoint = "raise_remote_exception" requests_mock.post( re.compile(f"mock://example.com/{endpoint}"), content=_exception_response( exception=Exception(error_message), status_code=status_code, - old_exception_schema=old_exception_schema, ), ) with pytest.raises(RemoteException) as exc_info: rpc_client._post(endpoint, data={}) assert str(exc_info.value.args[0]["type"]) == "Exception" assert str(exc_info.value.args[0]["message"]) == error_message + if status_code == 503: + assert isinstance(exc_info.value, TransientRemoteException) + else: + assert not isinstance(exc_info.value, TransientRemoteException) diff --git a/swh/core/api/tests/test_rpc_server.py b/swh/core/api/tests/test_rpc_server.py index dfe02af..9bf319c 100644 --- a/swh/core/api/tests/test_rpc_server.py +++ b/swh/core/api/tests/test_rpc_server.py @@ -1,150 +1,220 @@ -# Copyright (C) 2018-2019 The Software Heritage developers +# Copyright (C) 2018-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import json from flask import url_for import msgpack import pytest from swh.core.api import ( JSONFormatter, MsgpackFormatter, RPCServerApp, + encode_data_server, + error_handler, negotiate, remote_api_endpoint, ) from .test_serializers import ExtraType, extra_decoders, extra_encoders +class MyCustomException(Exception): + pass + + class MyRPCServerApp(RPCServerApp): extra_type_encoders = extra_encoders extra_type_decoders = extra_decoders class TestStorage: @remote_api_endpoint("test_endpoint_url") def endpoint_test(self, test_data, db=None, cur=None): assert test_data == "spam" return "egg" @remote_api_endpoint("path/to/endpoint") def something(self, data, db=None, cur=None): return data @remote_api_endpoint("serializer_test") def serializer_test(self, data, db=None, cur=None): assert data == ["foo", ExtraType("bar", b"baz")] return ExtraType({"spam": "egg"}, "qux") + @remote_api_endpoint("crashy/builtin") + def crashy(self, data, db=None, cur=None): + raise ValueError("this is an unexpected exception") + + @remote_api_endpoint("crashy/custom") + def custom_crashy(self, data, db=None, cur=None): + raise MyCustomException("try again later!") + @pytest.fixture def app(): - return MyRPCServerApp("testapp", backend_class=TestStorage) + app = MyRPCServerApp("testapp", backend_class=TestStorage) + + @app.errorhandler(MyCustomException) + def custom_error_handler(exception): + return error_handler(exception, encode_data_server, status_code=503) + + @app.errorhandler(Exception) + def default_error_handler(exception): + return error_handler(exception, encode_data_server) + + return app def test_api_rpc_server_app_ok(app): assert isinstance(app, MyRPCServerApp) actual_rpc_server2 = MyRPCServerApp( "app2", backend_class=TestStorage, backend_factory=TestStorage ) assert isinstance(actual_rpc_server2, MyRPCServerApp) actual_rpc_server3 = MyRPCServerApp("app3") assert isinstance(actual_rpc_server3, MyRPCServerApp) def test_api_rpc_server_app_misconfigured(): expected_error = "backend_factory should only be provided if backend_class is" with pytest.raises(ValueError, match=expected_error): MyRPCServerApp("failed-app", backend_factory="something-to-make-it-raise") def test_api_endpoint(flask_app_client): res = flask_app_client.post( url_for("something"), headers=[("Content-Type", "application/json"), ("Accept", "application/json")], data=json.dumps({"data": "toto"}), ) assert res.status_code == 200 assert res.mimetype == "application/json" def test_api_nego_default(flask_app_client): res = flask_app_client.post( url_for("something"), headers=[("Content-Type", "application/json")], data=json.dumps({"data": "toto"}), ) assert res.status_code == 200 assert res.mimetype == "application/json" assert res.data == b'"toto"' def test_api_nego_accept(flask_app_client): res = flask_app_client.post( url_for("something"), headers=[ ("Accept", "application/x-msgpack"), ("Content-Type", "application/x-msgpack"), ], data=msgpack.dumps({"data": "toto"}), ) assert res.status_code == 200 assert res.mimetype == "application/x-msgpack" assert res.data == b"\xa4toto" def test_rpc_server(flask_app_client): res = flask_app_client.post( url_for("endpoint_test"), headers=[ ("Content-Type", "application/x-msgpack"), ("Accept", "application/x-msgpack"), ], data=b"\x81\xa9test_data\xa4spam", ) assert res.status_code == 200 assert res.mimetype == "application/x-msgpack" assert res.data == b"\xa3egg" +def test_rpc_server_exception(flask_app_client): + res = flask_app_client.post( + url_for("crashy"), + headers=[ + ("Content-Type", "application/x-msgpack"), + ("Accept", "application/x-msgpack"), + ], + data=msgpack.dumps({"data": "toto"}), + ) + + assert res.status_code == 500 + assert res.mimetype == "application/x-msgpack", res.data + data = msgpack.loads(res.data) + assert ( + data.items() + >= { + "type": "ValueError", + "module": "builtins", + "args": ["this is an unexpected exception"], + }.items() + ), data + + +def test_rpc_server_custom_exception(flask_app_client): + res = flask_app_client.post( + url_for("custom_crashy"), + headers=[ + ("Content-Type", "application/x-msgpack"), + ("Accept", "application/x-msgpack"), + ], + data=msgpack.dumps({"data": "toto"}), + ) + + assert res.status_code == 503 + assert res.mimetype == "application/x-msgpack", res.data + data = msgpack.loads(res.data) + assert ( + data.items() + >= { + "type": "MyCustomException", + "module": "swh.core.api.tests.test_rpc_server", + "args": ["try again later!"], + }.items() + ), data + + def test_rpc_server_extra_serializers(flask_app_client): res = flask_app_client.post( url_for("serializer_test"), headers=[ ("Content-Type", "application/x-msgpack"), ("Accept", "application/x-msgpack"), ], data=b"\x81\xa4data\x92\xa3foo\x82\xc4\x07swhtype\xa9extratype" b"\xc4\x01d\x92\xa3bar\xc4\x03baz", ) assert res.status_code == 200 assert res.mimetype == "application/x-msgpack" assert res.data == ( b"\x82\xc4\x07swhtype\xa9extratype\xc4" b"\x01d\x92\x81\xa4spam\xa3egg\xa3qux" ) def test_api_negotiate_no_extra_encoders(app, flask_app_client): url = "/test/negotiate/no/extra/encoders" @app.route(url, methods=["POST"]) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def endpoint(): return "test" res = flask_app_client.post( url, headers=[("Content-Type", "application/json")], ) assert res.status_code == 200 assert res.mimetype == "application/json" assert res.data == b'"test"'