diff --git a/PKG-INFO b/PKG-INFO index 51b7b02..92247e9 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,39 +1,39 @@ Metadata-Version: 2.1 Name: swh.core -Version: 2.12 +Version: 2.13 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-core Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-core/ Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/x-rst Provides-Extra: testing-core Provides-Extra: logging Provides-Extra: db Provides-Extra: http Provides-Extra: github Provides-Extra: testing License-File: LICENSE License-File: AUTHORS Software Heritage - Core foundations ==================================== Low-level utilities and helpers used by almost all other modules in the stack. core library for swh's modules: - config parser - serialization - logging mechanism - database connection - http-based RPC client/server diff --git a/debian/changelog b/debian/changelog index da4c96f..c980034 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,1417 +1,1419 @@ -swh-core (2.12-1~swh2~bpo10+1) buster-swh; urgency=medium +swh-core (2.13-1~swh1) unstable-swh; urgency=medium - * Rebuild for buster-swh + * New upstream release 2.13 - (tagged by David Douard + on 2022-07-05 14:22:04 +0200) + * Upstream changes: - v2.13 - -- Software Heritage autobuilder (on jenkins-debian1) Mon, 20 Jun 2022 08:20:39 +0000 + -- Software Heritage autobuilder (on jenkins-debian1) Tue, 05 Jul 2022 12:38:24 +0000 swh-core (2.12-1~swh2) unstable-swh; urgency=medium * Bump new release with correct deps. -- Antoine R. Dumont (@ardumont) Mon, 20 Jun 2022 10:13:14 +0200 swh-core (2.12-1~swh1) unstable-swh; urgency=medium * New upstream release 2.12 - (tagged by Nicolas Dandrimont on 2022-06-16 13:38:52 +0200) * Upstream changes: - Release swh.core v2.12 - Add support for zstandard compressed tarballs -- Software Heritage autobuilder (on jenkins-debian1) Thu, 16 Jun 2022 11:42:04 +0000 swh-core (2.11-1~swh1) unstable-swh; urgency=medium * New upstream release 2.11 - (tagged by Antoine R. Dumont (@ardumont) on 2022-06-10 10:20:37 +0200) * Upstream changes: - v2.11 - tarball: Use standard Python module zipfile to extract jar archive - Add missing coverage on `swh db version` cli - db: Grant read access to guest user on all schema tables -- Software Heritage autobuilder (on jenkins-debian1) Fri, 10 Jun 2022 08:24:24 +0000 swh-core (2.10-1~swh1) unstable-swh; urgency=medium * New upstream release 2.10 - (tagged by Antoine R. Dumont (@ardumont) on 2022-06-02 16:03:41 +0200) * Upstream changes: - v2.10 - github/utils: Deal with exotic urls to canonicalize - deprecate the db/pytest_plugin.py module - move initialize_database_for_module in db_utils - mark postgresql_fact fixture factory function as deprecated - tests: use stock pytest_postgresql factory function - docs/db: Update datastore requirement -- Software Heritage autobuilder (on jenkins-debian1) Thu, 02 Jun 2022 14:39:11 +0000 swh-core (2.9.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.9.0 - (tagged by Antoine R. Dumont (@ardumont) on 2022-05-30 17:36:23 +0200) * Upstream changes: - v2.9.0 - Allow module to specify another config key than their module name - cli.db: Reword ignore sentence -- Software Heritage autobuilder (on jenkins-debian1) Mon, 30 May 2022 15:41:42 +0000 swh-core (2.8.1-1~swh1) unstable-swh; urgency=medium * New upstream release 2.8.1 - (tagged by Antoine R. Dumont (@ardumont) on 2022-05-30 17:06:11 +0200) * Upstream changes: - v2.8.1 - cli.db: Use attribute current_version instead of undeclared getter - cli.db: Fix help message typo -- Software Heritage autobuilder (on jenkins-debian1) Mon, 30 May 2022 15:10:17 +0000 swh-core (2.8.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.8.0 - (tagged by Antoine R. Dumont (@ardumont) on 2022-05-20 18:17:20 +0200) * Upstream changes: - v2.8.0 - Deal with git protocol url to canonicalize to https -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 May 2022 16:21:13 +0000 swh-core (2.7.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.7.0 - (tagged by Antoine R. Dumont (@ardumont) on 2022-05-20 15:26:30 +0200) * Upstream changes: - v2.7.0 - Use GitHubSession to make canonical computation deal with rate limit -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 May 2022 13:30:15 +0000 swh-core (2.6.0-1~swh2) unstable-swh; urgency=medium * Bump new release -- Antoine R. Dumont (@ardumont) Fri, 20 May 2022 10:26:03 +0200 swh-core (2.6.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.6.0 - (tagged by Antoine R. Dumont (@ardumont) on 2022-05-20 08:54:46 +0200) * Upstream changes: - v2.6.0 - Extract reusable github tests fixtures into its own pytest_plugin - test_db/test_db_copy_to: Fix hypothesis FailedHealthCheck error - Refactor swh.lister.github.utils to swh.core.github.utils - Add utility function to retrieve canonical github urls - Make 'python -m swh' work as cli entry point - db_utils: Make connect_to_conninfo use through contextmanager - Upgrade mypy to 0.942 to fix support of types-psycopg2 >= 2.9.12 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 May 2022 06:58:23 +0000 swh-core (2.5.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.5.0 - (tagged by Valentin Lorentz on 2022-04-25 13:51:26 +0200) * Upstream changes: - v2.5.0 - * Make db_transaction's client_options configurable at run time - * sentry: always override init settings with the environment variables - * Add support for disabling logging-based events in sentry - * RPC server: explicitly handle sentry exception capture (instead of gunicorn) - * statsd: add an error_type tag to @timed error counters - * cli: Ensure tests don't mess with the global logging setup - * mypy/pre-commit/pytest maintenance -- Software Heritage autobuilder (on jenkins-debian1) Mon, 25 Apr 2022 11:56:02 +0000 swh-core (2.4.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.4.0 - (tagged by Valentin Lorentz on 2022-03-29 14:39:15 +0200) * Upstream changes: - v2.4.0 - * Fix support of Werkzeug 2.1.0 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 29 Mar 2022 12:42:28 +0000 swh-core (2.3.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.3.0 - (tagged by David Douard on 2022-03-14 17:19:37 +0100) * Upstream changes: - v2.3.0 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 14 Mar 2022 16:23:52 +0000 swh-core (2.2.2-1~swh1) unstable-swh; urgency=medium * New upstream release 2.2.2 - (tagged by David Douard on 2022-03-09 17:44:25 +0100) * Upstream changes: - v2.2.2 - small fixes in the `swh db upgrade` command -- Software Heritage autobuilder (on jenkins-debian1) Wed, 09 Mar 2022 16:48:21 +0000 swh-core (2.2.1-1~swh1) unstable-swh; urgency=medium * New upstream release 2.2.1 - (tagged by Nicolas Dandrimont on 2022-03-03 20:20:11 +0100) * Upstream changes: - Release swh.core v2.2.1 - quiesce deprecation warnings for autogenerated RPC clients - fix typing for a few methods in db_utils -- Software Heritage autobuilder (on jenkins-debian1) Thu, 03 Mar 2022 19:24:59 +0000 swh-core (2.2.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.2.0 - (tagged by Valentin Lorentz on 2022-03-02 11:45:18 +0100) * Upstream changes: - v2.2.0 - * utils: Add a new 'iter_chunks' function -- Software Heritage autobuilder (on jenkins-debian1) Wed, 02 Mar 2022 10:47:47 +0000 swh-core (2.1.1-1~swh1) unstable-swh; urgency=medium * New upstream release 2.1.1 - (tagged by Valentin Lorentz on 2022-03-01 17:00:36 +0100) * Upstream changes: - v2.1.1 - * Fix wrong version numbers in deprecation message. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 01 Mar 2022 16:03:05 +0000 swh-core (2.1.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.1.0 - (tagged by Valentin Lorentz on 2022-03-01 16:58:02 +0100) * Upstream changes: - v2.1.0 - * RPCClient: Make methods {get,post}{,_stream} protected - * RPCServerApp: Add hooks to add behaviors to generated methods -- Software Heritage autobuilder (on jenkins-debian1) Tue, 01 Mar 2022 16:00:51 +0000 swh-core (2.0.0-2~swh1) unstable-swh; urgency=medium * Bump dependency constranints for pytest and pytest-postgresql. -- David Douard Wed, 23 Feb 2022 10:56:37 +0100 swh-core (2.0.0-1~swh1) unstable-swh; urgency=medium * New upstream release 2.0.0 - (tagged by David Douard on 2022-02-17 15:26:59 +0100) * Upstream changes: - v2.0.0 - add support for generic db version handling for postgresql backends, - add support for generic db upgrade (for postgresql backends), - upgrade pytest- postgresql based tests scaffolding to use the - template-based db creation (instead of the truncate-based db reset). -- Software Heritage autobuilder (on jenkins-debian1) Wed, 23 Feb 2022 09:34:41 +0000 swh-core (1.1.1-1~swh1) unstable-swh; urgency=medium * New upstream release 1.1.1 - (tagged by Valentin Lorentz on 2022-02-04 12:52:06 +0100) * Upstream changes: - v1.1.1 - * Require pytest to be <7.0.0 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 04 Feb 2022 11:54:45 +0000 swh-core (1.1.0-1~swh1) unstable-swh; urgency=medium * New upstream release 1.1.0 - (tagged by David Douard on 2022-01-20 15:39:02 +0100) * Upstream changes: - v1.1.0 - add a Statsd.status_gauge() context manager - add support for env var substitution in STATS_TAGS - pin mypy version and clean reauirements a bit -- Software Heritage autobuilder (on jenkins-debian1) Thu, 20 Jan 2022 14:44:14 +0000 swh-core (1.0.0-2~swh1) unstable-swh; urgency=medium * Add missing B-D on python3-blinker for sentry-sdk -- Nicolas Dandrimont Thu, 02 Dec 2021 12:34:50 +0100 swh-core (1.0.0-1~swh1) unstable-swh; urgency=medium * New upstream release 1.0.0 - (tagged by David Douard on 2021-12-02 10:51:10 +0100) * Upstream changes: - v1.0.0 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 02 Dec 2021 10:00:19 +0000 swh-core (0.15.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.15.1 - (tagged by Valentin Lorentz on 2021-11-08 14:10:08 +0100) * Upstream changes: - v0.15.1 - * Require pytest-postgresql < 4.0 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 08 Nov 2021 13:12:36 +0000 swh-core (0.15.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.15.0 - (tagged by Antoine R. Dumont (@ardumont) on 2021-09-20 11:45:26 +0200) * Upstream changes: - v0.15.0 - tarball: Fallback to guess archive format from mimetype when no format detected -- Software Heritage autobuilder (on jenkins-debian1) Mon, 20 Sep 2021 09:48:28 +0000 swh-core (0.14.6-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.6 - (tagged by Antoine Lambert on 2021-09-16 10:48:35 +0200) * Upstream changes: - version 0.14.6 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 16 Sep 2021 08:51:58 +0000 swh-core (0.14.5-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.5 - (tagged by Valentin Lorentz on 2021-08-30 10:31:47 +0200) * Upstream changes: - v0.14.5 - * tarball: Add support for .tbz, .tbz2, and .jar -- Software Heritage autobuilder (on jenkins-debian1) Mon, 30 Aug 2021 08:34:03 +0000 swh-core (0.14.4-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.4 - (tagged by Valentin Lorentz on 2021-07-30 16:44:26 +0200) * Upstream changes: - v0.14.4 - * add stream_results_optional -- Software Heritage autobuilder (on jenkins-debian1) Fri, 30 Jul 2021 14:47:25 +0000 swh-core (0.14.3-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.3 - (tagged by Antoine Lambert on 2021-06-11 15:41:38 +0200) * Upstream changes: - version 0.14.3 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 11 Jun 2021 13:47:13 +0000 swh-core (0.14.2-2~swh1) unstable-swh; urgency=medium * Rebuild v0.14.2 after missing unzip dependency -- Antoine Lambert Thu, 10 Jun 2021 16:24:09 +0200 swh-core (0.14.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.2 - (tagged by Antoine Lambert on 2021-06-10 16:09:06 +0200) * Upstream changes: - version 0.14.2 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 10 Jun 2021 14:13:25 +0000 swh-core (0.14.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.1 - (tagged by Valentin Lorentz on 2021-05-06 15:33:56 +0200) * Upstream changes: - v0.14.1 - * Fix reserved name being used in pytest plugin -- Software Heritage autobuilder (on jenkins-debian1) Thu, 06 May 2021 13:38:24 +0000 swh-core (0.14.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.14.0 - (tagged by Valentin Lorentz on 2021-05-06 14:14:10 +0200) * Upstream changes: - v0.14.0 - Add support for pytest- postgresql 3.0.0 - For consistency, I renamed db_name to dbname everywhere, so this will - affect other SWH packages. -- Software Heritage autobuilder (on jenkins-debian1) Thu, 06 May 2021 12:17:31 +0000 swh-core (0.13.3-1~swh1) unstable-swh; urgency=medium * New upstream release 0.13.3 - (tagged by Antoine Lambert on 2021-05-06 13:54:51 +0200) * Upstream changes: - version 0.13.3 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 06 May 2021 11:59:49 +0000 swh-core (0.13.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.13.2 - (tagged by Valentin Lorentz on 2021-05-06 10:40:51 +0200) * Upstream changes: - v0.13.2 - * tarball: properly normalize perms for all extracted files - * requirements-db-pytestplugin: Don't install pytest-postgresql 3.0+ -- Software Heritage autobuilder (on jenkins-debian1) Thu, 06 May 2021 08:44:45 +0000 swh-core (0.13.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.13.1 - (tagged by Antoine Lambert on 2021-04-29 14:21:29 +0200) * Upstream changes: - version 0.13.1 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 29 Apr 2021 12:25:15 +0000 swh-core (0.13.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.13.0 - (tagged by Vincent SELLIER on 2021-04-06 19:01:37 +0200) * Upstream changes: - v0.13.0 - Support several backends on RPCServerApp -- Software Heritage autobuilder (on jenkins-debian1) Tue, 06 Apr 2021 17:07:10 +0000 swh-core (0.12.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.12.1 - (tagged by Valentin Lorentz on 2021-04-06 12:34:34 +0200) * Upstream changes: - v0.12.1 - * tests: Drop hypothesis < 6 requirement - * README.rst: Remove getting-started instructions, they are duplicates - * Improve/fix documentation of requests_mock_datadir - * Remove dependency on 'decorator' (fixes a regression in decorator 5.0.5) -- Software Heritage autobuilder (on jenkins-debian1) Tue, 06 Apr 2021 10:37:17 +0000 swh-core (0.12.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.12.0 - (tagged by David Douard on 2021-02-16 11:48:58 +0100) * Upstream changes: - v0.12.0 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 16 Feb 2021 10:52:14 +0000 swh-core (0.11.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.11.0 - (tagged by David Douard on 2020-12-08 15:35:05 +0100) * Upstream changes: - v0.11.0 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 08 Dec 2020 14:38:23 +0000 swh-core (0.10.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.10.0 - (tagged by Nicolas Dandrimont on 2020-12-02 11:46:38 +0100) * Upstream changes: - Release swh.core 0.10.0 - db.tests.db_testing: Drop unused database test utilities. - api.serializers: Add support for serializing large negative integers with - msgpack. - Do not mutate api.serializers.ENCODERS or DECODERS. - swh cli: Add support for setting up the log level of multiple loggers. -- Software Heritage autobuilder (on jenkins-debian1) Wed, 02 Dec 2020 10:50:35 +0000 swh-core (0.9.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.9.1 - (tagged by Antoine Lambert on 2020-11-23 11:24:50 +0100) * Upstream changes: - version 0.9.1 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 23 Nov 2020 10:29:15 +0000 swh-core (0.9.0-1~swh2) unstable-swh; urgency=medium * Split packages python3-swh.core and python3-swh.core.db.pytestplugin -- Antoine R. Dumont (@ardumont) Fri, 20 Nov 2020 16:37:00 +0000 swh-core (0.9.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.9.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-11-20 15:35:53 +0100) * Upstream changes: - v0.9.0 - Clarify names around the swh.core.db.pytest_plugin tests - setup: Separate pytest- postgresql dependency and declare it when needed - RPCClient: Fix reraise_exceptions regression - api/serializers: Add Exception type encoder and decoder - Makefile.local: Ensure all tests are executed when invoking make test - core.db.cli: Add coverage and ensure `swh db *` works as expected - core tests: disambiguate arg 'request' through typing -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 Nov 2020 14:37:00 +0000 swh-core (0.8.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.8.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-30 09:12:17 +0100) * Upstream changes: - v0.8.0 - cli.db: Open init-admin subcmd to initialize superuser-level scripts -- Software Heritage autobuilder (on jenkins-debian1) Fri, 30 Oct 2020 08:13:02 +0000 swh-core (0.7.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.7.1 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-22 18:51:16 +0200) * Upstream changes: - v0.7.1 - Move SWHDatabaseJanitor to db.pytest_plugin module -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Oct 2020 16:51:59 +0000 swh-core (0.7.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.7.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-22 18:31:18 +0200) * Upstream changes: - v0.7.0 - remote_api_endpoint: Allow to declare what http method to use - api.RPCServerApp: Adapt sync rpc server with async rpc server -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Oct 2020 16:32:04 +0000 swh-core (0.6.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.6.1 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-22 13:53:41 +0200) * Upstream changes: - v0.6.1 - Move pytest_plugin declaration to top-level conftest -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Oct 2020 11:54:49 +0000 swh-core (0.6.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.6.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-22 13:24:55 +0200) * Upstream changes: - v0.6.0 - asynchronous.RPCServerApp: Align implementation with api.RPCServerApp -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Oct 2020 11:25:29 +0000 swh-core (0.5.0-1~swh2) unstable-swh; urgency=medium * Bump new release with dependency updated -- Antoine R. Dumont thu, 22 Oct 2020 11:21:04 +0200 swh-core (0.5.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.5.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-22 11:11:29 +0200) * Upstream changes: - v0.5.0 - Install postgresql_fact fixture for faster postgres tests - api.tests.test_async: Simplify fixture setup - core.config: Drop no longer used SWHConfig -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Oct 2020 09:12:14 +0000 swh-core (0.4.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.4.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-02 11:44:10 +0200) * Upstream changes: - v0.4.0 - config: Deprecate SWHConfig in favor of load_from_envvar function -- Software Heritage autobuilder (on jenkins-debian1) Fri, 02 Oct 2020 09:44:53 +0000 swh-core (0.3.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.1 - (tagged by Valentin Lorentz on 2020-10-01 12:38:08 +0200) * Upstream changes: - v0.3.1 - * Add specific celery task arguments to metadata sent to systemd-journald - * SortedList: Don't inherit from UserList. -- Software Heritage autobuilder (on jenkins-debian1) Thu, 01 Oct 2020 10:40:27 +0000 swh-core (0.3.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.0 - (tagged by David Douard on 2020-09-23 16:24:40 +0200) * Upstream changes: - v0.3.0 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 23 Sep 2020 14:27:08 +0000 swh-core (0.2.3-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.3 - (tagged by Valentin Lorentz on 2020-08-17 13:55:41 +0200) * Upstream changes: - v0.2.3 - * tarball: add test for permissions. - * Move SortedList from swh-storage. -- Software Heritage autobuilder (on jenkins-debian1) Mon, 17 Aug 2020 11:57:43 +0000 swh-core (0.2.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.2 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-31 13:41:19 +0200) * Upstream changes: - v0.2.2 - api.classes: Open swh.core.api.classes.stream_results -- Software Heritage autobuilder (on jenkins-debian1) Fri, 31 Jul 2020 11:44:33 +0000 swh-core (0.2.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.1 - (tagged by Valentin Lorentz on 2020-07-30 19:16:57 +0200) * Upstream changes: - v0.2.1 - Make @remote_api_endpoint preserve typing information for mypy. -- Software Heritage autobuilder (on jenkins-debian1) Thu, 30 Jul 2020 17:20:05 +0000 swh-core (0.2.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-29 11:18:37 +0200) * Upstream changes: - v0.2.0 - core.api: Expose serializable PagedResult object for pagination api - test_serializers: Refactor using pytest - Migrate from vcversioner to setuptools- scm -- Software Heritage autobuilder (on jenkins-debian1) Wed, 29 Jul 2020 09:20:56 +0000 swh-core (0.1.2-1~swh1) unstable-swh; urgency=medium * New upstream release 0.1.2 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-08 13:41:51 +0200) * Upstream changes: - v0.1.2 - test_serializers: Move to pytest for that specific erratic assertion -- Software Heritage autobuilder (on jenkins-debian1) Wed, 08 Jul 2020 11:43:25 +0000 swh-core (0.1.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.1.1 - (tagged by Antoine R. Dumont (@ardumont) on 2020-07-08 13:03:52 +0200) * Upstream changes: - v0.1.1 - api.tests: Fix unsupported matches keyword to match - requirements-db: Move typing- extension from test to runtime deps -- Software Heritage autobuilder (on jenkins-debian1) Wed, 08 Jul 2020 11:07:55 +0000 swh-core (0.1.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.1.0 - (tagged by David Douard on 2020-07-06 14:33:28 +0200) * Upstream changes: - v0.1.0 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 06 Jul 2020 12:36:15 +0000 swh-core (0.0.95-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.95 - (tagged by Nicolas Dandrimont on 2020-04-17 17:20:35 +0200) * Upstream changes: - Release swh.core v0.0.95 - support serializing large integers in msgpack - add documentation for CLI - move formatting to black -- Software Heritage autobuilder (on jenkins-debian1) Fri, 17 Apr 2020 16:05:01 +0000 swh-core (0.0.94-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.94 - (tagged by Valentin Lorentz on 2020-02-28 14:28:11 +0100) * Upstream changes: - v0.0.94 - Allow subclasses of RPCClient to override methods. -- Software Heritage autobuilder (on jenkins-debian1) Fri, 28 Feb 2020 13:32:41 +0000 swh-core (0.0.93-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.93 - (tagged by Valentin Lorentz on 2020-02-26 15:26:29 +0100) * Upstream changes: - v0.0.93 - Reintroduce support for decoding legacy msgpack encoding -- Software Heritage autobuilder (on jenkins-debian1) Wed, 26 Feb 2020 14:29:46 +0000 swh-core (0.0.92-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.92 - (tagged by Valentin Lorentz on 2020-02-19 15:40:57 +0100) * Upstream changes: - v0.0.92 - Add support for msgpack 1.0.0 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 19 Feb 2020 14:45:27 +0000 swh-core (0.0.91-1~swh2) unstable-swh; urgency=medium * Add missing python3-iso8601 build dependency -- Antoine Lambert Wed, 19 Feb 2020 11:18:34 +0100 swh-core (0.0.91-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.91 - (tagged by Antoine Lambert on 2020-02-18 16:43:59 +0100) * Upstream changes: - version 0.0.91 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 18 Feb 2020 15:51:39 +0000 swh-core (0.0.90-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.90 - (tagged by Valentin Lorentz on 2020-02-18 13:54:30 +0100) * Upstream changes: - v0.0.90 - Remove exception pickling from the async server, as in the sync server. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 18 Feb 2020 12:59:38 +0000 swh-core (0.0.89-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.89 - (tagged by Valentin Lorentz on 2020-02-18 11:35:01 +0100) * Upstream changes: - v0.0.89 - * Change msgpack serialization to be closer to the JSON one. - * Add support for extra {de,en}coders. - * Add extra_type_encoders and extra_type_decoders attributes to RPC clients and servers. - * Use iso8601.parse_date instead of dateutil.parser.parse. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 18 Feb 2020 10:43:34 +0000 swh-core (0.0.88-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.88 - (tagged by Valentin Lorentz on 2020-02-14 12:22:23 +0100) * Upstream changes: - v0.0.88 - In case of errors, return a simple dictionary instead of pickled exception. -- Software Heritage autobuilder (on jenkins-debian1) Fri, 14 Feb 2020 11:27:02 +0000 swh-core (0.0.87-1~swh2) unstable-swh; urgency=medium * Fix package build -- Antoine R. Dumont (@ardumont) Thu, 30 Jan 2020 14:06:54 +0100 swh-core (0.0.87-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.87 - (tagged by Valentin Lorentz on 2020-01-29 12:21:48 +0100) * Upstream changes: - v0.0.87 - Make db_transaction* remove db/cur from the signature. -- Software Heritage autobuilder (on jenkins-debian1) Wed, 29 Jan 2020 11:28:23 +0000 swh-core (0.0.86-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.86 - (tagged by Antoine R. Dumont (@ardumont) on 2020-01-23 09:08:56 +0100) * Upstream changes: - v0.0.86 - sentry: Add environment variable $SWH_SENTRY_ENVIRONMENT - pytest_plugin: Fix sphinx warnings -- Software Heritage autobuilder (on jenkins-debian1) Thu, 23 Jan 2020 08:12:40 +0000 swh-core (0.0.85-1~swh2) unstable-swh; urgency=medium * Fix package build on buster -- Antoine Lambert Thu, 16 Jan 2020 11:00:00 +0000 swh-core (0.0.85-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.85 - (tagged by Valentin Lorentz on 2020-01-15 13:07:36 +0100) * Upstream changes: - v0.0.85 - Add env var SWH_MAIN_PACKAGE -- Software Heritage autobuilder (on jenkins-debian1) Wed, 15 Jan 2020 12:11:49 +0000 swh-core (0.0.84-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.84 - (tagged by Antoine R. Dumont (@ardumont) on 2019-12-13 09:08:00 +0100) * Upstream changes: - v0.0.84 - Improve tarball support for tar.lz, tar.x, tar.Z files -- Software Heritage autobuilder (on jenkins-debian1) Fri, 13 Dec 2019 08:14:23 +0000 swh-core (0.0.83-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.83 - (tagged by Antoine R. Dumont (@ardumont) on 2019-12-12 16:03:25 +0100) * Upstream changes: - v0.0.83 - core.config: Rename configuration key -- Software Heritage autobuilder (on jenkins-debian1) Thu, 12 Dec 2019 15:06:28 +0000 swh-core (0.0.82-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.82 - (tagged by Nicolas Dandrimont on 2019-12-11 15:04:09 +0100) * Upstream changes: - Release swh.core 0.0.82 - Add missing conftest.py to MANIFEST.in -- Software Heritage autobuilder (on jenkins-debian1) Wed, 11 Dec 2019 14:09:00 +0000 swh-core (0.0.81-1~swh2) unstable-swh; urgency=medium * Add dependency to python3-sentry-sdk -- Nicolas Dandrimont Wed, 11 Dec 2019 14:58:54 +0100 swh-core (0.0.81-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.81 - (tagged by Valentin Lorentz on 2019-12-10 13:59:17 +0100) * Upstream changes: - v0.0.81 - * Include all requirements in MANIFEST.in - * Split test requirements to try and properly minimize dependencies - * Make the CLI initialize sentry-sdk based on CLI options/envvars. - * Add gunicorn config script to initialize sentry-sdk based on envvars. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 10 Dec 2019 13:03:13 +0000 swh-core (0.0.80-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.80 - (tagged by Nicolas Dandrimont on 2019-11-19 16:36:35 +0100) * Upstream changes: - Release swh.core v0.0.80 - Let TypeErrors pass through the RPC layer - Register SIGINT/SIGTERM handlers for the CLI -- Software Heritage autobuilder (on jenkins-debian1) Tue, 19 Nov 2019 15:41:09 +0000 swh-core (0.0.79-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.79 - (tagged by Stefano Zacchiroli on 2019-11-18 13:35:19 +0100) * Upstream changes: - v0.0.79 ======= - * RPCClient: add response attribute to RemoteException - * RPCClent: rename and refactor check_status (now raise_for_status) - * RPCClient: check HTTP status code for errors also when streaming - * cli: Add support for loading a logging configuration file - * cli: Allow adding a Notes section between Options and Commands - * Add trailing dot to help texts for consistency - * logger: only flatten dicts if all keys are strings - * Move to @pytest.fixture from yield_fixture - * test_rpc_client_server.py: fix typo in docstring -- Software Heritage autobuilder (on jenkins-debian1) Mon, 18 Nov 2019 12:42:12 +0000 swh-core (0.0.78-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.78 - (tagged by Nicolas Dandrimont on 2019-11-06 18:01:56 +0100) * Upstream changes: - Release swh.core 0.0.78 - allow the swh command to work even when a plugin fails - hardcode bytea and bytea[] type oids in BaseDb -- Software Heritage autobuilder (on jenkins-debian1) Wed, 06 Nov 2019 17:05:38 +0000 swh-core (0.0.77-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.77 - (tagged by Antoine R. Dumont (@ardumont) on 2019-11-06 14:10:58 +0100) * Upstream changes: - v0.0.77 - pytest_plugin: Decode url to resolve filename - api/serializers: Force json module use to decode requests text response -- Software Heritage autobuilder (on jenkins-debian1) Wed, 06 Nov 2019 13:15:03 +0000 swh-core (0.0.76-1~swh2) unstable-swh; urgency=medium * Force using the swh.core pytest plugin -- Nicolas Dandrimont Wed, 23 Oct 2019 14:50:04 +0200 swh-core (0.0.76-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.76 - (tagged by Nicolas Dandrimont on 2019-10-18 10:16:20 +0200) * Upstream changes: - Release swh.core v0.0.76 - Make the systemd dependency optional -- Software Heritage autobuilder (on jenkins-debian1) Fri, 18 Oct 2019 08:25:15 +0000 swh-core (0.0.75-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.75 - (tagged by Antoine R. Dumont (@ardumont) on 2019-10-14 17:51:58 +0200) * Upstream changes: - v0.0.75 - pytest_plugin: Add support for http requests -- Software Heritage autobuilder (on jenkins-debian1) Mon, 14 Oct 2019 15:57:05 +0000 swh-core (0.0.74-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.74 - (tagged by David Douard on 2019-10-11 15:30:51 +0200) * Upstream changes: - v0.0.74 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 11 Oct 2019 13:35:17 +0000 swh-core (0.0.73-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.73 - (tagged by Antoine R. Dumont (@ardumont) on 2019-10-09 14:16:04 +0200) * Upstream changes: - v0.0.73 - Improve pytest-plugin fixture to ease testing with pagination -- Software Heritage autobuilder (on jenkins-debian1) Wed, 09 Oct 2019 12:20:36 +0000 swh-core (0.0.72-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.72 - (tagged by Antoine R. Dumont (@ardumont) on 2019-10-09 10:59:28 +0200) * Upstream changes: - v0.0.72 - Fix tox.ini's py3 environment -- Software Heritage autobuilder (on jenkins-debian1) Wed, 09 Oct 2019 09:02:51 +0000 swh-core (0.0.70-1~swh2) unstable-swh; urgency=medium * Add new dependency on python3-tz -- Nicolas Dandrimont Tue, 01 Oct 2019 15:07:09 +0200 swh-core (0.0.70-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.70 - (tagged by Stefano Zacchiroli on 2019-09-27 10:16:41 +0200) * Upstream changes: - v0.0.70 - init.py: switch to documented way of extending path -- Software Heritage autobuilder (on jenkins-debian1) Fri, 27 Sep 2019 08:21:29 +0000 swh-core (0.0.69-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.69 - (tagged by Stefano Zacchiroli on 2019-09-20 15:50:52 +0200) * Upstream changes: - v0.0.69 - MANIFEST.in: ship py.typed -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 Sep 2019 13:54:15 +0000 swh-core (0.0.68-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.68 - (tagged by Stefano Zacchiroli on 2019-09-20 15:05:29 +0200) * Upstream changes: - v0.0.68 - * mypy: ignore django-stubs, needed only by hypothesis - * mypy: use conffile to ignore requests_mock - * typing: minimal changes to make a no-op mypy run pass - * db_testing.py: do not explode when TEST_DB_DUMP = None - * swh.core.config.parse_config_file: fix sphinx markup in docstring - * statsd: protect access to the statsd's socket - * tests: add tests for swh.logger and swh.tarball modules - * Remove fallback when aiohttp_utils is not installed. -- Software Heritage autobuilder (on jenkins-debian1) Fri, 20 Sep 2019 13:09:50 +0000 swh-core (0.0.67-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.67 - (tagged by Valentin Lorentz on 2019-08-22 13:56:36 +0200) * Upstream changes: - v0.0.67 - Improve error handling in Db.copy_to -- Software Heritage autobuilder (on jenkins-debian1) Thu, 22 Aug 2019 12:02:14 +0000 swh-core (0.0.66-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.66 - (tagged by David Douard on 2019-07-30 13:55:16 +0200) * Upstream changes: - v0.0.66 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 30 Jul 2019 11:58:47 +0000 swh-core (0.0.65-1~swh2) unstable-swh; urgency=medium * debian/control: add missing dependencies. -- David Douard Tue, 16 Jul 2019 14:46:43 +0200 swh-core (0.0.65-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.65 - (tagged by David Douard on 2019-07-15 16:49:47 +0200) * Upstream changes: - v0.0.65 - needed to fix my mess with 0.0.64 tag, since the wrong 0.0.64 version has - already been pushed to pypi. -- Software Heritage autobuilder (on jenkins-debian1) Mon, 15 Jul 2019 14:53:29 +0000 swh-core (0.0.64-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.64 - (tagged by David Douard on 2019-07-15 16:33:32 +0200) * Upstream changes: - v0.0.64 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 15 Jul 2019 14:37:04 +0000 swh-core (0.0.63-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.63 - (tagged by Antoine Lambert on 2019-05-21 13:12:11 +0200) * Upstream changes: - version 0.0.63 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 21 May 2019 11:15:45 +0000 swh-core (0.0.62-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.62 - (tagged by Antoine Lambert on 2019-05-20 14:56:05 +0200) * Upstream changes: - version 0.0.62 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 20 May 2019 13:01:38 +0000 swh-core (0.0.61-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.61 - (tagged by David Douard on 2019-05-17 10:32:07 +0200) * Upstream changes: - v0.0.61 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 17 May 2019 08:38:08 +0000 swh-core (0.0.60-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.60 - (tagged by David Douard on 2019-05-06 15:27:44 +0200) * Upstream changes: - v0.0.60 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 06 May 2019 13:32:48 +0000 swh-core (0.0.59-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.59 - (tagged by Valentin Lorentz on 2019-04-09 16:55:41 +0200) * Upstream changes: - Explicitly give Db connections back to the pool. - So they gracefully release the connection on error instead - of relying on reference-counting to call the Db's `__del__` - (which does not happen in Hypothesis tests) because a ref - to it is kept via the traceback object. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 09 Apr 2019 16:12:32 +0000 swh-core (0.0.58-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.58 - (tagged by Antoine Lambert on 2019-04-02 17:19:05 +0200) * Upstream changes: - version 0.0.58 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 02 Apr 2019 15:24:34 +0000 swh-core (0.0.57-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.57 - (tagged by Nicolas Dandrimont on 2019-03-28 15:51:27 +0100) * Upstream changes: - Release swh.core v0.0.57 - Move to native async primitives - Fix statsd.timed exceptional behavior bug/misfeature - Fix SWHRemoteAPI post_stream method -- Software Heritage autobuilder (on jenkins-debian1) Thu, 28 Mar 2019 14:55:58 +0000 swh-core (0.0.56-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.56 - (tagged by David Douard on 2019-03-19 10:17:06 +0100) * Upstream changes: - v0.0.56 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 19 Mar 2019 09:27:18 +0000 swh-core (0.0.55-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.55 - (tagged by Antoine R. Dumont (@ardumont) on 2019-02-19 12:28:26 +0100) * Upstream changes: - v0.0.55 - Fix runtime dependencies -- Software Heritage autobuilder (on jenkins-debian1) Tue, 19 Feb 2019 11:32:28 +0000 swh-core (0.0.54-1~swh2) unstable-swh; urgency=medium * New upstream release 0.0.54 * Upstream changes: - Add missing build dependencies -- Antoine R. Dumont (@ardumont) Tue, 12 Feb 2019 16:25:34 +0000 swh-core (0.0.54-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.54 - (tagged by Valentin Lorentz on 2019-02-11 16:47:18 +0100) * Upstream changes: - Add test for BaseDb.connect. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 12 Feb 2019 12:37:43 +0000 swh-core (0.0.53-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.53 - (tagged by Antoine R. Dumont (@ardumont) on 2019-02-08 09:09:30 +0100) * Upstream changes: - v0.0.53 - Fix debian build -- Software Heritage autobuilder (on jenkins-debian1) Fri, 08 Feb 2019 08:12:31 +0000 swh-core (0.0.52-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.52 - (tagged by David Douard on 2019-02-06 15:24:04 +0100) * Upstream changes: - v0.0.52 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 06 Feb 2019 14:27:14 +0000 swh-core (0.0.51-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.51 - (tagged by David Douard on 2019-02-01 14:28:27 +0100) * Upstream changes: - v0.0.51 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 01 Feb 2019 13:31:45 +0000 swh-core (0.0.50-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.50 - (tagged by Nicolas Dandrimont on 2019-01-09 15:50:58 +0100) * Upstream changes: - Release swh.core v0.0.50 - Add statsd client module - Log used config files -- Software Heritage autobuilder (on jenkins-debian1) Wed, 09 Jan 2019 14:54:37 +0000 swh-core (0.0.49-1~swh1) unstable-swh; urgency=medium * Make DbTestFixture.setUp() accept and pass *args and **kwargs. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 08 Jan 2019 16:38:02 +0000 swh-core (0.0.48-1~swh1) unstable-swh; urgency=medium * v0.0.48 * swh.core.cli: Update swh-db-init to make it idemtpotent -- Antoine R. Dumont (@ardumont) Tue, 08 Jan 2019 15:33:15 +0000 swh-core (0.0.47-1~swh1) unstable-swh; urgency=medium * v0.0.47 * swh.core.cli: Fix flag -- Antoine R. Dumont (@ardumont) Tue, 08 Jan 2019 15:16:09 +0000 swh-core (0.0.46-1~swh1) unstable-swh; urgency=medium * v0.0.46 * utils.grouper: Improve implementation * Remove now-obsolete information about swh.core.worker -- Antoine R. Dumont (@ardumont) Tue, 08 Jan 2019 14:37:34 +0000 swh-core (0.0.45-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.45 * Compatibility with recent msgpack * Debian packaging-related cleanups -- Nicolas Dandrimont Thu, 22 Nov 2018 21:09:53 +0100 swh-core (0.0.44-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.44 * Refactor the database testing fixtures * Stop unsafe serialization/deserialization constructs * Update tests to use nose -- Nicolas Dandrimont Thu, 18 Oct 2018 18:20:12 +0200 swh-core (0.0.43-1~swh1) unstable-swh; urgency=medium * v0.0.43 * Fix missing dependency declaration -- Antoine R. Dumont (@ardumont) Thu, 11 Oct 2018 15:47:06 +0200 swh-core (0.0.42-1~swh1) unstable-swh; urgency=medium * v0.0.42 * Fix missing dependency declaration -- Antoine R. Dumont (@ardumont) Thu, 11 Oct 2018 15:45:25 +0200 swh-core (0.0.41-1~swh1) unstable-swh; urgency=medium * Add functions to generate HTTP API clients and servers from databases. * Summary: This moves the interesting parts of D505 into the core, so other components can use them as well. * Test Plan: `make test` * Reviewers: ardumont, seirl, #reviewers * Reviewed By: ardumont, #reviewers * Subscribers: douardda * Differential Revision: https://forge.softwareheritage.org/D507 -- Valentin Lorentz Thu, 11 Oct 2018 10:57:27 +0200 swh-core (0.0.40-1~swh1) unstable-swh; urgency=medium * v0.0.40 * swh.core.api.SWHRemoteAPI: Permit to set a query timeout option -- Antoine R. Dumont (@ardumont) Thu, 24 May 2018 12:10:03 +0200 swh-core (0.0.39-1~swh1) unstable-swh; urgency=medium * v0.0.39 * package: Add missing runtime dependency -- Antoine R. Dumont (@ardumont) Thu, 26 Apr 2018 15:24:22 +0200 swh-core (0.0.38-1~swh1) unstable-swh; urgency=medium * v0.0.38 * tests: Use more reasonable psql options for db restores * swh.core.serializers: Add custom types serialization -- Antoine R. Dumont (@ardumont) Thu, 26 Apr 2018 15:15:27 +0200 swh-core (0.0.37-1~swh1) unstable-swh; urgency=medium * v0.0.37 * Move test fixture in swh.core.tests.server_testing module -- Antoine R. Dumont (@ardumont) Wed, 25 Apr 2018 15:00:02 +0200 swh-core (0.0.36-1~swh1) unstable-swh; urgency=medium * v0.0.36 * Migrate swh.loader.tar.tarball module in swh.core -- Antoine R. Dumont (@ardumont) Wed, 06 Dec 2017 12:03:29 +0100 swh-core (0.0.35-1~swh1) unstable-swh; urgency=medium * Release swh.core version 0.0.35 * Update packaging runes -- Nicolas Dandrimont Thu, 12 Oct 2017 18:07:50 +0200 swh-core (0.0.34-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.34 * New modular database test fixture -- Nicolas Dandrimont Mon, 07 Aug 2017 18:29:48 +0200 swh-core (0.0.33-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.33 * Be more conservative with remote API responses -- Nicolas Dandrimont Mon, 19 Jun 2017 19:01:38 +0200 swh-core (0.0.32-1~swh1) unstable-swh; urgency=medium * Release swh-core v0.0.32 * Add asynchronous streaming methods for internal APIs * Remove task arguments from systemd-journal loggers -- Nicolas Dandrimont Tue, 09 May 2017 14:04:22 +0200 swh-core (0.0.31-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.31 * Add explicit dependency on python3-systemd -- Nicolas Dandrimont Fri, 07 Apr 2017 15:11:26 +0200 swh-core (0.0.30-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.30 * drop swh.core.hashutil (moved to swh.model.hashutil) * add a systemd logger -- Nicolas Dandrimont Fri, 07 Apr 2017 11:49:15 +0200 swh-core (0.0.29-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.29 * Catch proper exception in the base API client -- Nicolas Dandrimont Thu, 02 Feb 2017 00:19:25 +0100 swh-core (0.0.28-1~swh1) unstable-swh; urgency=medium * v0.0.28 * Refactoring some common code into swh.core -- Antoine R. Dumont (@ardumont) Thu, 26 Jan 2017 14:54:22 +0100 swh-core (0.0.27-1~swh1) unstable-swh; urgency=medium * v0.0.27 * Fix issue with default boolean value -- Antoine R. Dumont (@ardumont) Thu, 20 Oct 2016 16:15:20 +0200 swh-core (0.0.26-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.26 * Raise an exception when a configuration file exists and is unreadable -- Nicolas Dandrimont Wed, 12 Oct 2016 10:16:09 +0200 swh-core (0.0.25-1~swh1) unstable-swh; urgency=medium * v0.0.25 * Add new function utils.cwd -- Antoine R. Dumont (@ardumont) Thu, 29 Sep 2016 21:29:37 +0200 swh-core (0.0.24-1~swh1) unstable-swh; urgency=medium * v0.0.24 * Deal with edge case in logger regarding json -- Antoine R. Dumont (@ardumont) Thu, 22 Sep 2016 12:21:09 +0200 swh-core (0.0.23-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.23 * Properly fix the PyYAML dependency -- Nicolas Dandrimont Tue, 23 Aug 2016 16:20:29 +0200 swh-core (0.0.22-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.22 * Proper loading of yaml and ini files in all paths -- Nicolas Dandrimont Fri, 19 Aug 2016 15:45:55 +0200 swh-core (0.0.21-1~swh1) unstable-swh; urgency=medium * v0.0.21 * Update test tools -- Antoine R. Dumont (@ardumont) Tue, 19 Jul 2016 14:47:01 +0200 swh-core (0.0.20-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.20 * Add some generic bytes <-> escaped unicode methods -- Nicolas Dandrimont Tue, 14 Jun 2016 16:54:41 +0200 swh-core (0.0.19-1~swh1) unstable-swh; urgency=medium * v0.0.19 * Resurrect swh.core.utils -- Antoine R. Dumont (@ardumont) Fri, 15 Apr 2016 12:40:43 +0200 swh-core (0.0.18-1~swh1) unstable-swh; urgency=medium * v0.0.18 * Add swh.core.utils * serializers: support UUIDs all around -- Antoine R. Dumont (@ardumont) Sat, 26 Mar 2016 11:16:33 +0100 swh-core (0.0.17-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.17 * Allow serialization of UUIDs -- Nicolas Dandrimont Fri, 04 Mar 2016 11:40:56 +0100 swh-core (0.0.16-1~swh1) unstable-swh; urgency=medium * Release swh.core version 0.0.16 * add bytehex_to_hash and hash_to_bytehex in hashutil * move scheduling utilities to swh.scheduler -- Nicolas Dandrimont Fri, 19 Feb 2016 18:12:10 +0100 swh-core (0.0.15-1~swh1) unstable-swh; urgency=medium * Release v0.0.15 * Add hashutil.hash_git_object -- Nicolas Dandrimont Wed, 16 Dec 2015 16:31:26 +0100 swh-core (0.0.14-1~swh1) unstable-swh; urgency=medium * v0.0.14 * Add simple README * Update license * swh.core.hashutil.hashfile can now deal with filepath as bytes -- Antoine R. Dumont (@ardumont) Fri, 23 Oct 2015 11:13:14 +0200 swh-core (0.0.13-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.13 -- Nicolas Dandrimont Fri, 09 Oct 2015 17:32:49 +0200 swh-core (0.0.12-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.12 -- Nicolas Dandrimont Tue, 06 Oct 2015 17:34:34 +0200 swh-core (0.0.11-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.11 -- Nicolas Dandrimont Sat, 03 Oct 2015 15:57:03 +0200 swh-core (0.0.10-1~swh1) unstable-swh; urgency=medium * Prepare deploying swh.core v0.0.10 -- Nicolas Dandrimont Sat, 03 Oct 2015 12:28:52 +0200 swh-core (0.0.9-1~swh1) unstable-swh; urgency=medium * Prepare deploying swh.core v0.0.9 -- Nicolas Dandrimont Sat, 03 Oct 2015 11:36:55 +0200 swh-core (0.0.8-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.8 -- Nicolas Dandrimont Thu, 01 Oct 2015 12:31:44 +0200 swh-core (0.0.7-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.7 -- Nicolas Dandrimont Thu, 01 Oct 2015 11:29:04 +0200 swh-core (0.0.6-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.6 -- Nicolas Dandrimont Tue, 29 Sep 2015 16:48:44 +0200 swh-core (0.0.5-1~swh1) unstable-swh; urgency=medium * Prepare v0.0.5 deployment -- Nicolas Dandrimont Tue, 29 Sep 2015 16:08:32 +0200 swh-core (0.0.4-1~swh1) unstable-swh; urgency=medium * Tagging swh.core 0.0.4 -- Nicolas Dandrimont Fri, 25 Sep 2015 15:41:26 +0200 swh-core (0.0.3-1~swh1) unstable-swh; urgency=medium * Tag swh.core v0.0.3 -- Nicolas Dandrimont Fri, 25 Sep 2015 11:07:10 +0200 swh-core (0.0.2-1~swh1) unstable-swh; urgency=medium * Deploy v0.0.2 -- Nicolas Dandrimont Wed, 23 Sep 2015 12:08:50 +0200 swh-core (0.0.1-1~swh1) unstable-swh; urgency=medium * Initial release * Tag v0.0.1 for deployment -- Nicolas Dandrimont Tue, 22 Sep 2015 14:52:26 +0200 diff --git a/swh.core.egg-info/PKG-INFO b/swh.core.egg-info/PKG-INFO index 51b7b02..92247e9 100644 --- a/swh.core.egg-info/PKG-INFO +++ b/swh.core.egg-info/PKG-INFO @@ -1,39 +1,39 @@ Metadata-Version: 2.1 Name: swh.core -Version: 2.12 +Version: 2.13 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-core Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-core/ Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/x-rst Provides-Extra: testing-core Provides-Extra: logging Provides-Extra: db Provides-Extra: http Provides-Extra: github Provides-Extra: testing License-File: LICENSE License-File: AUTHORS Software Heritage - Core foundations ==================================== Low-level utilities and helpers used by almost all other modules in the stack. core library for swh's modules: - config parser - serialization - logging mechanism - database connection - http-based RPC client/server diff --git a/swh.core.egg-info/SOURCES.txt b/swh.core.egg-info/SOURCES.txt index a2de7a2..2cd17f0 100644 --- a/swh.core.egg-info/SOURCES.txt +++ b/swh.core.egg-info/SOURCES.txt @@ -1,143 +1,145 @@ .git-blame-ignore-revs .gitignore .pre-commit-config.yaml AUTHORS CODE_OF_CONDUCT.md CONTRIBUTORS LICENSE MANIFEST.in Makefile Makefile.local README.rst conftest.py mypy.ini pyproject.toml pytest.ini requirements-db-pytestplugin.txt requirements-db.txt requirements-github.txt requirements-http.txt requirements-logging.txt requirements-swh.txt requirements-test.txt requirements.txt setup.cfg setup.py tox.ini docs/.gitignore docs/Makefile docs/README.rst docs/cli.rst docs/conf.py docs/db.rst docs/index.rst docs/_static/.placeholder docs/_templates/.placeholder swh/__init__.py swh/__main__.py swh.core.egg-info/PKG-INFO swh.core.egg-info/SOURCES.txt swh.core.egg-info/dependency_links.txt swh.core.egg-info/entry_points.txt swh.core.egg-info/requires.txt swh.core.egg-info/top_level.txt swh/core/__init__.py swh/core/api_async.py swh/core/collections.py swh/core/config.py swh/core/logger.py swh/core/py.typed swh/core/pytest_plugin.py swh/core/sentry.py swh/core/statsd.py swh/core/tarball.py swh/core/utils.py swh/core/api/__init__.py swh/core/api/asynchronous.py swh/core/api/classes.py swh/core/api/gunicorn_config.py swh/core/api/negotiation.py swh/core/api/serializers.py swh/core/api/tests/__init__.py swh/core/api/tests/conftest.py swh/core/api/tests/server_testing.py swh/core/api/tests/test_async.py swh/core/api/tests/test_classes.py swh/core/api/tests/test_gunicorn.py swh/core/api/tests/test_init.py swh/core/api/tests/test_rpc_client.py swh/core/api/tests/test_rpc_client_server.py swh/core/api/tests/test_rpc_server.py swh/core/api/tests/test_rpc_server_asynchronous.py swh/core/api/tests/test_serializers.py swh/core/cli/__init__.py swh/core/cli/db.py swh/core/db/__init__.py swh/core/db/common.py swh/core/db/db_utils.py swh/core/db/pytest_plugin.py swh/core/db/sql/35-dbversion.sql swh/core/db/sql/36-dbmodule.sql swh/core/db/tests/__init__.py swh/core/db/tests/conftest.py swh/core/db/tests/test_cli.py swh/core/db/tests/test_db.py swh/core/db/tests/test_db_utils.py swh/core/db/tests/data/cli/sql/0-superuser-init.sql +swh/core/db/tests/data/cli/sql/15-flavor.sql swh/core/db/tests/data/cli/sql/30-schema.sql swh/core/db/tests/data/cli/sql/40-funcs.sql swh/core/db/tests/data/cli/sql/50-data.sql swh/core/db/tests/data/cli_new/sql/0-superuser-init.sql +swh/core/db/tests/data/cli_new/sql/15-flavor.sql swh/core/db/tests/data/cli_new/sql/30-schema.sql swh/core/db/tests/data/cli_new/sql/40-funcs.sql swh/core/db/tests/data/cli_new/sql/50-data.sql swh/core/db/tests/data/cli_new/sql/upgrades/001.sql swh/core/db/tests/data/cli_new/sql/upgrades/002.sql swh/core/db/tests/data/cli_new/sql/upgrades/003.sql swh/core/db/tests/data/cli_new/sql/upgrades/004.sql swh/core/db/tests/data/cli_new/sql/upgrades/005.sql swh/core/db/tests/data/cli_new/sql/upgrades/006.sql swh/core/db/tests/pytest_plugin/__init__.py swh/core/db/tests/pytest_plugin/test_pytest_plugin.py swh/core/db/tests/pytest_plugin/data/0-schema.sql swh/core/db/tests/pytest_plugin/data/1-data.sql swh/core/github/__init__.py swh/core/github/pytest_plugin.py swh/core/github/utils.py swh/core/github/tests/__init__.py swh/core/github/tests/test_github_utils.py swh/core/github/tests/test_pytest_plugin.py swh/core/tests/__init__.py swh/core/tests/test_cli.py swh/core/tests/test_collections.py swh/core/tests/test_config.py swh/core/tests/test_logger.py swh/core/tests/test_pytest_plugin.py swh/core/tests/test_sentry.py swh/core/tests/test_statsd.py swh/core/tests/test_tarball.py swh/core/tests/test_utils.py swh/core/tests/data/archives/ca-certificates-20210603-1-any.pkg.tar.zst swh/core/tests/data/archives/groff-1.02.tar.Z swh/core/tests/data/archives/hello.jar swh/core/tests/data/archives/hello.tar swh/core/tests/data/archives/hello.tar.bz2 swh/core/tests/data/archives/hello.tar.gz swh/core/tests/data/archives/hello.tar.lz swh/core/tests/data/archives/hello.tar.x swh/core/tests/data/archives/hello.tbz swh/core/tests/data/archives/hello.tbz2 swh/core/tests/data/archives/hello.zip swh/core/tests/data/archives/msk316src.zip swh/core/tests/data/archives/tokei-12.1.2.crate swh/core/tests/data/http_example.com/something.json swh/core/tests/data/https_example.com/file.json swh/core/tests/data/https_example.com/file.json,name=doe,firstname=jane swh/core/tests/data/https_example.com/file.json_visit1 swh/core/tests/data/https_example.com/other.json swh/core/tests/data/https_forge.s.o/api_diffusion,attachments[uris]=1 swh/core/tests/data/https_www.reference.com/web,q=What+Is+an+Example+of+a+URL?,qo=contentPageRelatedSearch,o=600605,l=dir,sga=1 swh/core/tests/fixture/__init__.py swh/core/tests/fixture/conftest.py swh/core/tests/fixture/test_pytest_plugin.py swh/core/tests/fixture/data/https_example.com/file.json \ No newline at end of file diff --git a/swh/core/api/serializers.py b/swh/core/api/serializers.py index 37dad4b..091054d 100644 --- a/swh/core/api/serializers.py +++ b/swh/core/api/serializers.py @@ -1,325 +1,315 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import base64 import datetime from enum import Enum import json import traceback import types from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union from uuid import UUID import iso8601 import msgpack from requests import Response from swh.core.api.classes import PagedResult def encode_datetime(dt: datetime.datetime) -> str: """Wrapper of datetime.datetime.isoformat() that forbids naive datetimes.""" if dt.tzinfo is None: raise TypeError("can not serialize naive 'datetime.datetime' object") return dt.isoformat() def _encode_paged_result(obj: PagedResult) -> Dict[str, Any]: """Serialize PagedResult to a Dict.""" return { "results": obj.results, "next_page_token": obj.next_page_token, } def _decode_paged_result(obj: Dict[str, Any]) -> PagedResult: """Deserialize Dict into PagedResult""" return PagedResult( results=obj["results"], next_page_token=obj["next_page_token"], ) def exception_to_dict(exception: Exception) -> Dict[str, Any]: tb = traceback.format_exception(None, exception, exception.__traceback__) exc_type = type(exception) return { "type": exc_type.__name__, "module": exc_type.__module__, "args": exception.args, "message": str(exception), "traceback": tb, } def dict_to_exception(exc_dict: Dict[str, Any]) -> Exception: temp = __import__(exc_dict["module"], fromlist=[exc_dict["type"]]) try: return getattr(temp, exc_dict["type"])(*exc_dict["args"]) except Exception: # custom Exception type cannot be rebuilt, fallback to base Exception type return Exception(exc_dict["message"]) def encode_timedelta(td: datetime.timedelta) -> Dict[str, int]: return { "days": td.days, "seconds": td.seconds, "microseconds": td.microseconds, } ENCODERS: List[Tuple[type, str, Callable]] = [ (UUID, "uuid", str), (datetime.timedelta, "timedelta", encode_timedelta), (PagedResult, "paged_result", _encode_paged_result), (Exception, "exception", exception_to_dict), ] JSON_ENCODERS: List[Tuple[type, str, Callable]] = [ (datetime.datetime, "datetime", encode_datetime), (bytes, "bytes", lambda o: base64.b85encode(o).decode("ascii")), ] DECODERS: Dict[str, Callable] = { "timedelta": lambda d: datetime.timedelta(**d), "uuid": UUID, "paged_result": _decode_paged_result, "exception": dict_to_exception, # for BW compat, to be moved in JSON_DECODERS ASAP "datetime": lambda d: iso8601.parse_date(d, default_timezone=None), } JSON_DECODERS: Dict[str, Callable] = { "bytes": base64.b85decode, } def get_encoders( extra_encoders: Optional[List[Tuple[Type, str, Callable]]], with_json: bool = False ) -> List[Tuple[Type, str, Callable]]: encoders = ENCODERS if with_json: encoders = [*encoders, *JSON_ENCODERS] if extra_encoders: encoders = [*encoders, *extra_encoders] return encoders def get_decoders( extra_decoders: Optional[Dict[str, Callable]], with_json: bool = False ) -> Dict[str, Callable]: decoders = DECODERS if with_json: decoders = {**decoders, **JSON_DECODERS} if extra_decoders is not None: decoders = {**decoders, **extra_decoders} return decoders class MsgpackExtTypeCodes(Enum): LONG_INT = 1 LONG_NEG_INT = 2 def encode_data_client(data: Any, extra_encoders=None) -> bytes: try: return msgpack_dumps(data, extra_encoders=extra_encoders) except OverflowError as e: raise ValueError("Limits were reached. Please, check your input.\n" + str(e)) def decode_response(response: Response, extra_decoders=None) -> Any: content_type = response.headers["content-type"] if content_type.startswith("application/x-msgpack"): r = msgpack_loads(response.content, extra_decoders=extra_decoders) elif content_type.startswith("application/json"): r = json_loads(response.text, extra_decoders=extra_decoders) elif content_type.startswith("text/"): r = response.text else: raise ValueError("Wrong content type `%s` for API response" % content_type) return r class SWHJSONEncoder(json.JSONEncoder): """JSON encoder for data structures generated by Software Heritage. This JSON encoder extends the default Python JSON encoder and adds awareness for the following specific types: - bytes (get encoded as a Base85 string); - datetime.datetime (get encoded as an ISO8601 string). Non-standard types get encoded as a a dictionary with two keys: - swhtype with value 'bytes' or 'datetime'; - d containing the encoded value. SWHJSONEncoder also encodes arbitrary iterables as a list (allowing serialization of generators). Caveats: Limitations in the JSONEncoder extension mechanism prevent us from "escaping" dictionaries that only contain the swhtype and d keys, and therefore arbitrary data structures can't be round-tripped through SWHJSONEncoder and SWHJSONDecoder. """ def __init__(self, extra_encoders=None, **kwargs): super().__init__(**kwargs) self.encoders = get_encoders(extra_encoders, with_json=True) def default(self, o: Any) -> Union[Dict[str, Union[Dict[str, int], str]], list]: for (type_, type_name, encoder) in self.encoders: if isinstance(o, type_): return { "swhtype": type_name, "d": encoder(o), } try: return super().default(o) except TypeError as e: try: iterable = iter(o) except TypeError: raise e from None else: return list(iterable) class SWHJSONDecoder(json.JSONDecoder): """JSON decoder for data structures encoded with SWHJSONEncoder. This JSON decoder extends the default Python JSON decoder, allowing the decoding of: - bytes (encoded as a Base85 string); - datetime.datetime (encoded as an ISO8601 string). Non-standard types must be encoded as a a dictionary with exactly two keys: - swhtype with value 'bytes' or 'datetime'; - d containing the encoded value. To limit the impact our encoding, if the swhtype key doesn't contain a known value, the dictionary is decoded as-is. """ def __init__(self, extra_decoders=None, **kwargs): super().__init__(**kwargs) self.decoders = get_decoders(extra_decoders, with_json=True) def decode_data(self, o: Any) -> Any: if isinstance(o, dict): if set(o.keys()) == {"d", "swhtype"}: if o["swhtype"] == "bytes": return base64.b85decode(o["d"]) decoder = self.decoders.get(o["swhtype"]) if decoder: return decoder(self.decode_data(o["d"])) return {key: self.decode_data(value) for key, value in o.items()} if isinstance(o, list): return [self.decode_data(value) for value in o] else: return o def raw_decode(self, s: str, idx: int = 0) -> Tuple[Any, int]: data, index = super().raw_decode(s, idx) return self.decode_data(data), index def json_dumps(data: Any, extra_encoders=None) -> str: return json.dumps(data, cls=SWHJSONEncoder, extra_encoders=extra_encoders) def json_loads(data: str, extra_decoders=None) -> Any: return json.loads(data, cls=SWHJSONDecoder, extra_decoders=extra_decoders) def msgpack_dumps(data: Any, extra_encoders=None) -> bytes: """Write data as a msgpack stream""" encoders = get_encoders(extra_encoders) def encode_types(obj): if isinstance(obj, int): # integer overflowed while packing. Handle it as an extended type if obj > 0: code = MsgpackExtTypeCodes.LONG_INT.value else: code = MsgpackExtTypeCodes.LONG_NEG_INT.value obj = -obj length, rem = divmod(obj.bit_length(), 8) if rem: length += 1 return msgpack.ExtType(code, int.to_bytes(obj, length, "big")) if isinstance(obj, types.GeneratorType): return list(obj) for (type_, type_name, encoder) in encoders: if isinstance(obj, type_): return { b"swhtype": type_name, b"d": encoder(obj), } return obj return msgpack.packb( data, use_bin_type=True, datetime=True, # encode datetime as msgpack.Timestamp default=encode_types, ) def msgpack_loads(data: bytes, extra_decoders=None) -> Any: """Read data as a msgpack stream. .. Caution:: This function is used by swh.journal to decode the contents of the journal. This function **must** be kept backwards-compatible. """ decoders = get_decoders(extra_decoders) def ext_hook(code, data): if code == MsgpackExtTypeCodes.LONG_INT.value: return int.from_bytes(data, "big") elif code == MsgpackExtTypeCodes.LONG_NEG_INT.value: return -int.from_bytes(data, "big") raise ValueError("Unknown msgpack extended code %s" % code) def decode_types(obj): # Support for current encodings if set(obj.keys()) == {b"d", b"swhtype"}: decoder = decoders.get(obj[b"swhtype"]) if decoder: return decoder(obj[b"d"]) # Fallthrough return obj - try: - try: - return msgpack.unpackb( - data, - raw=False, - object_hook=decode_types, - ext_hook=ext_hook, - strict_map_key=False, - timestamp=3, # convert Timestamp in datetime objects (tz UTC) - ) - except TypeError: # msgpack < 0.6.0 - return msgpack.unpackb( - data, raw=False, object_hook=decode_types, ext_hook=ext_hook - ) - except TypeError: # msgpack < 0.5.2 - return msgpack.unpackb( - data, encoding="utf-8", object_hook=decode_types, ext_hook=ext_hook - ) + return msgpack.unpackb( + data, + raw=False, + object_hook=decode_types, + ext_hook=ext_hook, + strict_map_key=False, + timestamp=3, # convert Timestamp in datetime objects (tz UTC) + ) diff --git a/swh/core/db/db_utils.py b/swh/core/db/db_utils.py index ea66a1a..e325434 100644 --- a/swh/core/db/db_utils.py +++ b/swh/core/db/db_utils.py @@ -1,700 +1,702 @@ # Copyright (C) 2015-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from contextlib import contextmanager from datetime import datetime, timezone import functools from importlib import import_module import logging from os import path import pathlib import re import subprocess from typing import Collection, Dict, Iterator, List, Optional, Tuple, Union, cast import psycopg2 import psycopg2.errors import psycopg2.extensions from psycopg2.extensions import connection as pgconnection from psycopg2.extensions import encodings as pgencodings from psycopg2.extensions import make_dsn from psycopg2.extensions import parse_dsn as _parse_dsn from swh.core.utils import numfile_sortkey as sortkey logger = logging.getLogger(__name__) def now(): return datetime.now(tz=timezone.utc) def stored_procedure(stored_proc): """decorator to execute remote stored procedure, specified as argument Generally, the body of the decorated function should be empty. If it is not, the stored procedure will be executed first; the function body then. """ def wrap(meth): @functools.wraps(meth) def _meth(self, *args, **kwargs): cur = kwargs.get("cur", None) self._cursor(cur).execute("SELECT %s()" % stored_proc) meth(self, *args, **kwargs) return _meth return wrap def jsonize(value): """Convert a value to a psycopg2 JSON object if necessary""" if isinstance(value, dict): return psycopg2.extras.Json(value) return value @contextmanager def connect_to_conninfo( db_or_conninfo: Union[str, pgconnection] ) -> Iterator[pgconnection]: """Connect to the database passed as argument. Args: db_or_conninfo: A database connection, or a database connection info string Returns: a connected database handle or None if the database is not initialized """ if isinstance(db_or_conninfo, pgconnection): yield db_or_conninfo else: if "=" not in db_or_conninfo and "//" not in db_or_conninfo: # Database name db_or_conninfo = f"dbname={db_or_conninfo}" try: db = psycopg2.connect(db_or_conninfo) except psycopg2.Error: logger.exception("Failed to connect to `%s`", db_or_conninfo) else: yield db def swh_db_version(db_or_conninfo: Union[str, pgconnection]) -> Optional[int]: """Retrieve the swh version of the database. If the database is not initialized, this logs a warning and returns None. Args: db_or_conninfo: A database connection, or a database connection info string Returns: Either the version of the database, or None if it couldn't be detected """ try: with connect_to_conninfo(db_or_conninfo) as db: if not db: return None with db.cursor() as c: query = "select version from dbversion order by dbversion desc limit 1" try: c.execute(query) result = c.fetchone() if result: return result[0] except psycopg2.errors.UndefinedTable: return None except Exception: logger.exception("Could not get version from `%s`", db_or_conninfo) return None def swh_db_versions( db_or_conninfo: Union[str, pgconnection] ) -> Optional[List[Tuple[int, datetime, str]]]: """Retrieve the swh version history of the database. If the database is not initialized, this logs a warning and returns None. Args: db_or_conninfo: A database connection, or a database connection info string Returns: Either the version of the database, or None if it couldn't be detected """ try: with connect_to_conninfo(db_or_conninfo) as db: if not db: return None with db.cursor() as c: query = ( "select version, release, description " "from dbversion order by dbversion desc" ) try: c.execute(query) return cast(List[Tuple[int, datetime, str]], c.fetchall()) except psycopg2.errors.UndefinedTable: return None except Exception: logger.exception("Could not get versions from `%s`", db_or_conninfo) return None def swh_db_upgrade( conninfo: str, modname: str, to_version: Optional[int] = None ) -> int: """Upgrade the database at `conninfo` for module `modname` This will run migration scripts found in the `sql/upgrades` subdirectory of the module `modname`. By default, this will upgrade to the latest declared version. Args: conninfo: A database connection, or a database connection info string modname: datastore module the database stores content for to_version: if given, update the database to this version rather than the latest """ if to_version is None: to_version = 99999999 db_module, db_version, db_flavor = get_database_info(conninfo) if db_version is None: raise ValueError("Unable to retrieve the current version of the database") if db_module is None: raise ValueError("Unable to retrieve the module of the database") if db_module != modname: raise ValueError( "The stored module of the database is different than the given one" ) sqlfiles = [ fname for fname in get_sql_for_package(modname, upgrade=True) if db_version < int(fname.stem) <= to_version ] if not sqlfiles: return db_version for sqlfile in sqlfiles: new_version = int(path.splitext(path.basename(sqlfile))[0]) logger.info("Executing migration script '%s'", sqlfile) if db_version is not None and (new_version - db_version) > 1: logger.error( f"There are missing migration steps between {db_version} and " f"{new_version}. It might be expected but it most unlikely is not. " "Will stop here." ) return db_version execute_sqlfiles([sqlfile], conninfo, db_flavor) # check if the db version has been updated by the upgrade script db_version = swh_db_version(conninfo) assert db_version is not None if db_version == new_version: # nothing to do, upgrade script did the job pass elif db_version == new_version - 1: # it has not (new style), so do it swh_set_db_version( conninfo, new_version, desc=f"Upgraded to version {new_version} using {sqlfile}", ) db_version = swh_db_version(conninfo) else: # upgrade script did it wrong logger.error( f"The upgrade script {sqlfile} did not update the dbversion table " f"consistently ({db_version} vs. expected {new_version}). " "Will stop migration here. Please check your migration scripts." ) return db_version return new_version def swh_db_module(db_or_conninfo: Union[str, pgconnection]) -> Optional[str]: """Retrieve the swh module used to create the database. If the database is not initialized, this logs a warning and returns None. Args: db_or_conninfo: A database connection, or a database connection info string Returns: Either the module of the database, or None if it couldn't be detected """ try: with connect_to_conninfo(db_or_conninfo) as db: if not db: return None with db.cursor() as c: query = "select dbmodule from dbmodule limit 1" try: c.execute(query) resp = c.fetchone() if resp: return resp[0] except psycopg2.errors.UndefinedTable: return None except Exception: logger.exception("Could not get module from `%s`", db_or_conninfo) return None def swh_set_db_module( db_or_conninfo: Union[str, pgconnection], module: str, force=False ) -> None: """Set the swh module used to create the database. Fails if the dbmodule is already set or the table does not exist. Args: db_or_conninfo: A database connection, or a database connection info string module: the swh module to register (without the leading 'swh.') """ update = False if module.startswith("swh."): module = module[4:] current_module = swh_db_module(db_or_conninfo) if current_module is not None: if current_module == module: logger.warning("The database module is already set to %s", module) return if not force: raise ValueError( "The database module is already set to a value %s " "different than given %s", current_module, module, ) # force is True update = True with connect_to_conninfo(db_or_conninfo) as db: if not db: return None sqlfiles = [ fname for fname in get_sql_for_package("swh.core.db") if "dbmodule" in fname.stem ] execute_sqlfiles(sqlfiles, db_or_conninfo) with db.cursor() as c: if update: query = "update dbmodule set dbmodule = %s" else: query = "insert into dbmodule(dbmodule) values (%s)" c.execute(query, (module,)) db.commit() def swh_set_db_version( db_or_conninfo: Union[str, pgconnection], version: int, ts: Optional[datetime] = None, desc: str = "Work in progress", ) -> None: """Set the version of the database. Fails if the dbversion table does not exists. Args: db_or_conninfo: A database connection, or a database connection info string version: the version to add """ if ts is None: ts = now() with connect_to_conninfo(db_or_conninfo) as db: if not db: return None with db.cursor() as c: query = ( "insert into dbversion(version, release, description) " "values (%s, %s, %s)" ) c.execute(query, (version, ts, desc)) db.commit() def swh_db_flavor(db_or_conninfo: Union[str, pgconnection]) -> Optional[str]: """Retrieve the swh flavor of the database. If the database is not initialized, or the database doesn't support flavors, this returns None. Args: db_or_conninfo: A database connection, or a database connection info string Returns: The flavor of the database, or None if it could not be detected. """ try: with connect_to_conninfo(db_or_conninfo) as db: if not db: return None with db.cursor() as c: query = "select swh_get_dbflavor()" try: c.execute(query) result = c.fetchone() assert result is not None # to keep mypy happy return result[0] except psycopg2.errors.UndefinedFunction: # function not found: no flavor return None except Exception: logger.exception("Could not get flavor from `%s`", db_or_conninfo) return None # The following code has been imported from psycopg2, version 2.7.4, # https://github.com/psycopg/psycopg2/tree/5afb2ce803debea9533e293eef73c92ffce95bcd # and modified by Software Heritage. # # Original file: lib/extras.py # # psycopg2 is free software: you can redistribute it and/or modify it under the # terms of the GNU Lesser General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) any # later version. def _paginate(seq, page_size): """Consume an iterable and return it in chunks. Every chunk is at most `page_size`. Never return an empty chunk. """ page = [] it = iter(seq) while 1: try: for i in range(page_size): page.append(next(it)) yield page page = [] except StopIteration: if page: yield page return def _split_sql(sql): """Split *sql* on a single ``%s`` placeholder. Split on the %s, perform %% replacement and return pre, post lists of snippets. """ curr = pre = [] post = [] tokens = re.split(rb"(%.)", sql) for token in tokens: if len(token) != 2 or token[:1] != b"%": curr.append(token) continue if token[1:] == b"s": if curr is pre: curr = post else: raise ValueError("the query contains more than one '%s' placeholder") elif token[1:] == b"%": curr.append(b"%") else: raise ValueError( "unsupported format character: '%s'" % token[1:].decode("ascii", "replace") ) if curr is pre: raise ValueError("the query doesn't contain any '%s' placeholder") return pre, post def execute_values_generator(cur, sql, argslist, template=None, page_size=100): """Execute a statement using SQL ``VALUES`` with a sequence of parameters. Rows returned by the query are returned through a generator. You need to consume the generator for the queries to be executed! :param cur: the cursor to use to execute the query. :param sql: the query to execute. It must contain a single ``%s`` placeholder, which will be replaced by a `VALUES list`__. Example: ``"INSERT INTO mytable (id, f1, f2) VALUES %s"``. :param argslist: sequence of sequences or dictionaries with the arguments to send to the query. The type and content must be consistent with *template*. :param template: the snippet to merge to every item in *argslist* to compose the query. - If the *argslist* items are sequences it should contain positional placeholders (e.g. ``"(%s, %s, %s)"``, or ``"(%s, %s, 42)``" if there are constants value...). - If the *argslist* items are mappings it should contain named placeholders (e.g. ``"(%(id)s, %(f1)s, 42)"``). If not specified, assume the arguments are sequence and use a simple positional template (i.e. ``(%s, %s, ...)``), with the number of placeholders sniffed by the first element in *argslist*. :param page_size: maximum number of *argslist* items to include in every statement. If there are more items the function will execute more than one statement. :param yield_from_cur: Whether to yield results from the cursor in this function directly. .. __: https://www.postgresql.org/docs/current/static/queries-values.html After the execution of the function the `cursor.rowcount` property will **not** contain a total result. """ # we can't just use sql % vals because vals is bytes: if sql is bytes # there will be some decoding error because of stupid codec used, and Py3 # doesn't implement % on bytes. if not isinstance(sql, bytes): sql = sql.encode(pgencodings[cur.connection.encoding]) pre, post = _split_sql(sql) for page in _paginate(argslist, page_size=page_size): if template is None: template = b"(" + b",".join([b"%s"] * len(page[0])) + b")" parts = pre[:] for args in page: parts.append(cur.mogrify(template, args)) parts.append(b",") parts[-1:] = post cur.execute(b"".join(parts)) yield from cur def import_swhmodule(modname): if not modname.startswith("swh."): modname = f"swh.{modname}" try: m = import_module(modname) except ImportError as exc: logger.error(f"Could not load the {modname} module: {exc}") return None return m def get_sql_for_package(modname: str, upgrade: bool = False) -> List[pathlib.Path]: """Return the (sorted) list of sql script files for the given swh module If upgrade is True, return the list of available migration scripts, otherwise, return the list of initialization scripts. """ m = import_swhmodule(modname) if m is None: raise ValueError(f"Module {modname} cannot be loaded") sqldir = pathlib.Path(m.__file__).parent / "sql" if upgrade: sqldir /= "upgrades" if not sqldir.is_dir(): raise ValueError( "Module {} does not provide a db schema (no sql/ dir)".format(modname) ) return sorted(sqldir.glob("*.sql"), key=lambda x: sortkey(x.name)) def populate_database_for_package( modname: str, conninfo: str, flavor: Optional[str] = None ) -> Tuple[bool, Optional[int], Optional[str]]: """Populate the database, pointed at with ``conninfo``, using the SQL files found in the package ``modname``. Also fill the 'dbmodule' table with the given ``modname``. Args: modname: Name of the module of which we're loading the files conninfo: connection info string for the SQL database flavor: the module-specific flavor which we want to initialize the database under Returns: Tuple with three elements: whether the database has been initialized; the current version of the database; if it exists, the flavor of the database. """ current_version = swh_db_version(conninfo) if current_version is not None: dbflavor = swh_db_flavor(conninfo) return False, current_version, dbflavor def globalsortkey(key): "like sortkey but only on basenames" return sortkey(path.basename(key)) sqlfiles = get_sql_for_package(modname) + get_sql_for_package("swh.core.db") sqlfiles = sorted(sqlfiles, key=lambda x: sortkey(x.stem)) sqlfiles = [fpath for fpath in sqlfiles if "-superuser-" not in fpath.stem] execute_sqlfiles(sqlfiles, conninfo, flavor) # populate the dbmodule table swh_set_db_module(conninfo, modname) current_db_version = swh_db_version(conninfo) dbflavor = swh_db_flavor(conninfo) return True, current_db_version, dbflavor -def initialize_database_for_module(modname: str, version: int, **kwargs): +def initialize_database_for_module( + modname: str, version: int, flavor: Optional[str] = None, **kwargs +): """Helper function to initialize and populate a database for the given module This aims at helping the usage of pytest_postgresql for swh.core.db based datastores. Typical usage will be (here for swh.storage):: from pytest_postgresql import factories storage_postgresql_proc = factories.postgresql_proc( load=[partial(initialize_database_for_module, modname="storage", version=42)] ) storage_postgresql = factories.postgresql("storage_postgresql_proc") """ conninfo = psycopg2.connect(**kwargs).dsn init_admin_extensions(modname, conninfo) - populate_database_for_package(modname, conninfo) + populate_database_for_package(modname, conninfo, flavor) try: swh_set_db_version(conninfo, version) except psycopg2.errors.UniqueViolation: logger.warn( "Version already set by db init scripts. " f"This generally means the swh.{modname} package needs to be " "updated for swh.core>=1.2" ) def get_database_info( conninfo: str, ) -> Tuple[Optional[str], Optional[int], Optional[str]]: """Get version, flavor and module of the db""" dbmodule = swh_db_module(conninfo) dbversion = swh_db_version(conninfo) dbflavor = None if dbversion is not None: dbflavor = swh_db_flavor(conninfo) return (dbmodule, dbversion, dbflavor) def parse_dsn_or_dbname(dsn_or_dbname: str) -> Dict[str, str]: """Parse a psycopg2 dsn, falling back to supporting plain database names as well""" try: return _parse_dsn(dsn_or_dbname) except psycopg2.ProgrammingError: # psycopg2 failed to parse the DSN; it's probably a database name, # handle it as such return _parse_dsn(f"dbname={dsn_or_dbname}") def init_admin_extensions(modname: str, conninfo: str) -> None: """The remaining initialization process -- running -superuser- SQL files -- is done using the given conninfo, thus connecting to the newly created database """ sqlfiles = get_sql_for_package(modname) sqlfiles = [fname for fname in sqlfiles if "-superuser-" in fname.stem] execute_sqlfiles(sqlfiles, conninfo) def create_database_for_package( modname: str, conninfo: str, template: str = "template1" ): """Create the database pointed at with ``conninfo``, and initialize it using -superuser- SQL files found in the package ``modname``. Args: modname: Name of the module of which we're loading the files conninfo: connection info string or plain database name for the SQL database template: the name of the database to connect to and use as template to create the new database """ # Use the given conninfo string, but with dbname replaced by the template dbname # for the database creation step creation_dsn = parse_dsn_or_dbname(conninfo) dbname = creation_dsn["dbname"] creation_dsn["dbname"] = template logger.debug("db_create dbname=%s (from %s)", dbname, template) subprocess.check_call( [ "psql", "--quiet", "--no-psqlrc", "-v", "ON_ERROR_STOP=1", "-d", make_dsn(**creation_dsn), "-c", f'CREATE DATABASE "{dbname}"', ] ) init_admin_extensions(modname, conninfo) def execute_sqlfiles( sqlfiles: Collection[pathlib.Path], db_or_conninfo: Union[str, pgconnection], flavor: Optional[str] = None, ): """Execute a list of SQL files on the database pointed at with ``db_or_conninfo``. Args: sqlfiles: List of SQL files to execute db_or_conninfo: A database connection, or a database connection info string flavor: the database flavor to initialize """ if isinstance(db_or_conninfo, str): conninfo = db_or_conninfo else: conninfo = db_or_conninfo.dsn psql_command = [ "psql", "--quiet", "--no-psqlrc", "-v", "ON_ERROR_STOP=1", "-d", conninfo, ] flavor_set = False for sqlfile in sqlfiles: logger.debug(f"execute SQL file {sqlfile} dbname={conninfo}") subprocess.check_call(psql_command + ["-f", str(sqlfile)]) if ( flavor is not None and not flavor_set and sqlfile.name.endswith("-flavor.sql") ): logger.debug("Setting database flavor %s", flavor) query = f"insert into dbflavor (flavor) values ('{flavor}')" subprocess.check_call(psql_command + ["-c", query]) flavor_set = True if flavor is not None and not flavor_set: logger.warn( "Asked for flavor %s, but module does not support database flavors", flavor, ) # Grant read-access to guest user on all tables of the schema (if possible) with connect_to_conninfo(db_or_conninfo) as db: try: with db.cursor() as c: query = "grant select on all tables in schema public to guest" c.execute(query) except Exception: logger.warning("Grant read-only access to guest user failed. Skipping.") diff --git a/swh/core/db/tests/data/cli/sql/15-flavor.sql b/swh/core/db/tests/data/cli/sql/15-flavor.sql new file mode 100644 index 0000000..8cd1cdc --- /dev/null +++ b/swh/core/db/tests/data/cli/sql/15-flavor.sql @@ -0,0 +1,22 @@ +-- database flavor +create type database_flavor as enum ( + 'default', + 'flavorA', + 'flavorB' +); +comment on type database_flavor is 'Flavor of the current database'; + +create table dbflavor ( + flavor database_flavor, + single_row char(1) primary key default 'x', + check (single_row = 'x') +); +comment on table dbflavor is 'Database flavor storage'; +comment on column dbflavor.flavor is 'Database flavor currently deployed'; +comment on column dbflavor.single_row is 'Bogus column to force the table to have a single row'; + +create or replace function swh_get_dbflavor() returns database_flavor language sql stable as $$ + select coalesce((select flavor from dbflavor), 'default'); +$$; + +comment on function swh_get_dbflavor is 'Get the flavor of the database currently deployed'; diff --git a/swh/core/db/tests/data/cli_new/sql/15-flavor.sql b/swh/core/db/tests/data/cli_new/sql/15-flavor.sql new file mode 100644 index 0000000..8cd1cdc --- /dev/null +++ b/swh/core/db/tests/data/cli_new/sql/15-flavor.sql @@ -0,0 +1,22 @@ +-- database flavor +create type database_flavor as enum ( + 'default', + 'flavorA', + 'flavorB' +); +comment on type database_flavor is 'Flavor of the current database'; + +create table dbflavor ( + flavor database_flavor, + single_row char(1) primary key default 'x', + check (single_row = 'x') +); +comment on table dbflavor is 'Database flavor storage'; +comment on column dbflavor.flavor is 'Database flavor currently deployed'; +comment on column dbflavor.single_row is 'Bogus column to force the table to have a single row'; + +create or replace function swh_get_dbflavor() returns database_flavor language sql stable as $$ + select coalesce((select flavor from dbflavor), 'default'); +$$; + +comment on function swh_get_dbflavor is 'Get the flavor of the database currently deployed'; diff --git a/swh/core/db/tests/test_cli.py b/swh/core/db/tests/test_cli.py index a0c8a29..57d2b01 100644 --- a/swh/core/db/tests/test_cli.py +++ b/swh/core/db/tests/test_cli.py @@ -1,359 +1,361 @@ # Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import copy import os import traceback import pytest import yaml from swh.core.cli.db import db as swhdb from swh.core.db import BaseDb from swh.core.db.db_utils import import_swhmodule, swh_db_module, swh_db_version from swh.core.tests.test_cli import assert_section_contains def test_cli_swh_help(swhmain, cli_runner): swhmain.add_command(swhdb) result = cli_runner.invoke(swhmain, ["-h"]) assert result.exit_code == 0 assert_section_contains( result.output, "Commands", "db Software Heritage database generic tools." ) help_db_snippets = ( ( "Usage", ( "Usage: swh db [OPTIONS] COMMAND [ARGS]...", "Software Heritage database generic tools.", ), ), ( "Commands", ( "create Create a database for the Software Heritage .", "init Initialize a database for the Software Heritage .", "init-admin Execute superuser-level initialization steps", ), ), ) def test_cli_swh_db_help(swhmain, cli_runner): swhmain.add_command(swhdb) result = cli_runner.invoke(swhmain, ["db", "-h"]) assert result.exit_code == 0 for section, snippets in help_db_snippets: for snippet in snippets: assert_section_contains(result.output, section, snippet) @pytest.fixture def swh_db_cli(cli_runner, monkeypatch, postgresql): """This initializes a cli_runner and sets the correct environment variable expected by the cli to run appropriately (when not specifying the --dbname flag) """ db_params = postgresql.get_dsn_parameters() monkeypatch.setenv("PGHOST", db_params["host"]) monkeypatch.setenv("PGUSER", db_params["user"]) monkeypatch.setenv("PGPORT", db_params["port"]) return cli_runner, db_params def craft_conninfo(test_db, dbname=None) -> str: """Craft conninfo string out of the test_db object. This also allows to override the dbname.""" db_params = test_db.get_dsn_parameters() if dbname: params = copy.deepcopy(db_params) params["dbname"] = dbname else: params = db_params return "postgresql://{user}@{host}:{port}/{dbname}".format(**params) def test_cli_swh_db_create_and_init_db(cli_runner, postgresql, mock_import_swhmodule): """Create a db then initializing it should be ok""" module_name = "test.cli" conninfo = craft_conninfo(postgresql, "new-db") # This creates the db and installs the necessary admin extensions result = cli_runner.invoke(swhdb, ["create", module_name, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" # This initializes the schema and data result = cli_runner.invoke(swhdb, ["init", module_name, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" # the origin value in the scripts uses a hash function (which implementation wise # uses a function from the pgcrypt extension, installed during db creation step) with BaseDb.connect(conninfo).cursor() as cur: cur.execute("select * from origin") origins = cur.fetchall() assert len(origins) == 1 def test_cli_swh_db_initialization_fail_without_creation_first( cli_runner, postgresql, mock_import_swhmodule ): """Init command on an inexisting db cannot work""" module_name = "test.cli" # it's mocked here conninfo = craft_conninfo(postgresql, "inexisting-db") result = cli_runner.invoke(swhdb, ["init", module_name, "--dbname", conninfo]) # Fails because we cannot connect to an inexisting db assert result.exit_code == 1, f"Unexpected output: {result.output}" def test_cli_swh_db_initialization_fail_without_extension( cli_runner, postgresql, mock_import_swhmodule ): """Init command cannot work without privileged extension. In this test, the schema needs privileged extension to work. """ module_name = "test.cli" # it's mocked here conninfo = craft_conninfo(postgresql) result = cli_runner.invoke(swhdb, ["init", module_name, "--dbname", conninfo]) # Fails as the function `public.digest` is not installed, init-admin calls is needed # first (the next tests show such behavior) assert result.exit_code == 1, f"Unexpected output: {result.output}" def test_cli_swh_db_initialization_works_with_flags( cli_runner, postgresql, mock_import_swhmodule ): """Init commands with carefully crafted libpq conninfo works""" module_name = "test.cli" # it's mocked here conninfo = craft_conninfo(postgresql) result = cli_runner.invoke(swhdb, ["init-admin", module_name, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke(swhdb, ["init", module_name, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" # the origin values in the scripts uses a hash function (which implementation wise # uses a function from the pgcrypt extension, init-admin calls installs it) with BaseDb.connect(postgresql.dsn).cursor() as cur: cur.execute("select * from origin") origins = cur.fetchall() assert len(origins) == 1 def test_cli_swh_db_initialization_with_env( swh_db_cli, mock_import_swhmodule, postgresql ): """Init commands with standard environment variables works""" module_name = "test.cli" # it's mocked here cli_runner, db_params = swh_db_cli result = cli_runner.invoke( swhdb, ["init-admin", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( swhdb, ["init", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" # the origin values in the scripts uses a hash function (which implementation wise # uses a function from the pgcrypt extension, init-admin calls installs it) with BaseDb.connect(postgresql.dsn).cursor() as cur: cur.execute("select * from origin") origins = cur.fetchall() assert len(origins) == 1 def test_cli_swh_db_initialization_idempotent( swh_db_cli, mock_import_swhmodule, postgresql ): """Multiple runs of the init commands are idempotent""" module_name = "test.cli" # mocked cli_runner, db_params = swh_db_cli result = cli_runner.invoke( swhdb, ["init-admin", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( swhdb, ["init", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( swhdb, ["init-admin", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( swhdb, ["init", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" # the origin values in the scripts uses a hash function (which implementation wise # uses a function from the pgcrypt extension, init-admin calls installs it) with BaseDb.connect(postgresql.dsn).cursor() as cur: cur.execute("select * from origin") origins = cur.fetchall() assert len(origins) == 1 def test_cli_swh_db_create_and_init_db_new_api( cli_runner, postgresql, mock_import_swhmodule, mocker, tmp_path ): """Create a db then initializing it should be ok for a "new style" datastore""" module_name = "test.cli_new" conninfo = craft_conninfo(postgresql) # This initializes the schema and data cfgfile = tmp_path / "config.yml" cfgfile.write_text(yaml.dump({module_name: {"cls": "postgresql", "db": conninfo}})) result = cli_runner.invoke(swhdb, ["init-admin", module_name, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke(swhdb, ["-C", cfgfile, "init", module_name]) assert ( result.exit_code == 0 ), f"Unexpected output: {traceback.print_tb(result.exc_info[2])}" # the origin value in the scripts uses a hash function (which implementation wise # uses a function from the pgcrypt extension, installed during db creation step) with BaseDb.connect(conninfo).cursor() as cur: cur.execute("select * from origin") origins = cur.fetchall() assert len(origins) == 1 def test_cli_swh_db_upgrade_new_api(cli_runner, postgresql, datadir, mocker, tmp_path): """Upgrade scenario for a "new style" datastore""" module_name = "test.cli_new" # the `current_version` variable is the version that will be returned by # any call to `get_current_version()` in this test session, thanks to the # local mocked version of import_swhmodule() below. current_version = 1 # custom version of the mockup to make it easy to change the # current_version returned by get_current_version() # TODO: find a better solution for this... def import_swhmodule_mock(modname): if modname.startswith("test."): dirname = modname.split(".", 1)[1] def get_datastore(cls, **kw): return mocker.MagicMock(current_version=current_version) return mocker.MagicMock( __name__=modname, __file__=os.path.join(datadir, dirname, "__init__.py"), name=modname, get_datastore=get_datastore, ) return import_swhmodule(modname) mocker.patch("swh.core.db.db_utils.import_swhmodule", import_swhmodule_mock) conninfo = craft_conninfo(postgresql) # This initializes the schema and data cfgfile = tmp_path / "config.yml" cfgfile.write_text(yaml.dump({module_name: {"cls": "postgresql", "db": conninfo}})) result = cli_runner.invoke(swhdb, ["init-admin", module_name, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke(swhdb, ["-C", cfgfile, "init", module_name]) assert ( result.exit_code == 0 ), f"Unexpected output: {traceback.print_tb(result.exc_info[2])}" assert swh_db_version(conninfo) == 1 # the upgrade should not do anything because the datastore does advertise # version 1 result = cli_runner.invoke(swhdb, ["-C", cfgfile, "upgrade", module_name]) assert swh_db_version(conninfo) == 1 # advertise current version as 3, a simple upgrade should get us there, but # no further current_version = 3 result = cli_runner.invoke(swhdb, ["-C", cfgfile, "upgrade", module_name]) assert swh_db_version(conninfo) == 3 # an attempt to go further should not do anything result = cli_runner.invoke( swhdb, ["-C", cfgfile, "upgrade", module_name, "--to-version", 5] ) assert swh_db_version(conninfo) == 3 # an attempt to go lower should not do anything result = cli_runner.invoke( swhdb, ["-C", cfgfile, "upgrade", module_name, "--to-version", 2] ) assert swh_db_version(conninfo) == 3 # advertise current version as 6, an upgrade with --to-version 4 should # stick to the given version 4 and no further current_version = 6 result = cli_runner.invoke( swhdb, ["-C", cfgfile, "upgrade", module_name, "--to-version", 4] ) assert swh_db_version(conninfo) == 4 assert "migration was not complete" in result.output # attempt to upgrade to a newer version than current code version fails result = cli_runner.invoke( swhdb, ["-C", cfgfile, "upgrade", module_name, "--to-version", current_version + 1], ) assert result.exit_code != 0 assert swh_db_version(conninfo) == 4 cnx = BaseDb.connect(conninfo) with cnx.transaction() as cur: cur.execute("drop table dbmodule") assert swh_db_module(conninfo) is None # db migration should recreate the missing dbmodule table result = cli_runner.invoke(swhdb, ["-C", cfgfile, "upgrade", module_name]) assert result.exit_code == 0 assert "Warning: the database does not have a dbmodule table." in result.output assert ( "Write the module information (test.cli_new) in the database? [Y/n]" in result.output ) assert swh_db_module(conninfo) == module_name def test_cli_swh_db_version(swh_db_cli, mock_import_swhmodule, postgresql): module_name = "test.cli" cli_runner, db_params = swh_db_cli conninfo = craft_conninfo(postgresql, "test-db-version") # This creates the db and installs the necessary admin extensions result = cli_runner.invoke(swhdb, ["create", module_name, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" # This initializes the schema and data result = cli_runner.invoke(swhdb, ["init", module_name, "--dbname", conninfo]) actual_db_version = swh_db_version(conninfo) with BaseDb.connect(conninfo).cursor() as cur: cur.execute("select version from dbversion order by version desc limit 1") expected_version = cur.fetchone()[0] assert actual_db_version == expected_version assert result.exit_code == 0, f"Unexpected output: {result.output}" - assert f"initialized at version {expected_version}" in result.output + assert ( + f"initialized (flavor default) at version {expected_version}" in result.output + ) diff --git a/swh/core/db/tests/test_db_utils.py b/swh/core/db/tests/test_db_utils.py index 0a633f3..3808098 100644 --- a/swh/core/db/tests/test_db_utils.py +++ b/swh/core/db/tests/test_db_utils.py @@ -1,185 +1,208 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime, timedelta from os import path import pytest from swh.core.cli.db import db as swhdb from swh.core.db import BaseDb from swh.core.db.db_utils import ( get_database_info, get_sql_for_package, now, swh_db_module, swh_db_upgrade, swh_db_version, swh_db_versions, swh_set_db_module, ) from .test_cli import craft_conninfo @pytest.mark.parametrize("module", ["test.cli", "test.cli_new"]) def test_get_sql_for_package(mock_import_swhmodule, module): files = get_sql_for_package(module) assert files assert [f.name for f in files] == [ "0-superuser-init.sql", + "15-flavor.sql", "30-schema.sql", "40-funcs.sql", "50-data.sql", ] @pytest.mark.parametrize("module", ["test.cli", "test.cli_new"]) def test_db_utils_versions(cli_runner, postgresql, mock_import_swhmodule, module): """Check get_database_info, swh_db_versions and swh_db_module work ok This test checks db versions for both a db with "new style" set of sql init scripts (i.e. the dbversion table is not created in these scripts, but by the populate_database_for_package() function directly, via the 'swh db init' command) and an "old style" set (dbversion created in the scripts)S. """ conninfo = craft_conninfo(postgresql) result = cli_runner.invoke(swhdb, ["init-admin", module, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( swhdb, ["init", module, "--dbname", conninfo, "--initial-version", 10] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" # check the swh_db_module() function assert swh_db_module(conninfo) == module # the dbversion and dbmodule tables exists and are populated dbmodule, dbversion, dbflavor = get_database_info(conninfo) # check also the swh_db_versions() function versions = swh_db_versions(conninfo) assert dbmodule == module assert dbversion == 10 - assert dbflavor is None + assert dbflavor == "default" # check also the swh_db_versions() function versions = swh_db_versions(conninfo) assert len(versions) == 1 assert versions[0][0] == 10 if module == "test.cli": assert versions[0][1] == datetime.fromisoformat( "2016-02-22T15:56:28.358587+00:00" ) assert versions[0][2] == "Work In Progress" else: # new scheme but with no datastore (so no version support from there) assert versions[0][2] == "DB initialization" # add a few versions in dbversion cnx = BaseDb.connect(conninfo) with cnx.transaction() as cur: cur.executemany( "insert into dbversion(version, release, description) values (%s, %s, %s)", [(i, now(), f"Upgrade to version {i}") for i in range(11, 15)], ) dbmodule, dbversion, dbflavor = get_database_info(conninfo) assert dbmodule == module assert dbversion == 14 - assert dbflavor is None + assert dbflavor == "default" versions = swh_db_versions(conninfo) assert len(versions) == 5 for i, (version, ts, desc) in enumerate(versions): assert version == (14 - i) # these are in reverse order if version > 10: assert desc == f"Upgrade to version {version}" assert (now() - ts) < timedelta(seconds=1) @pytest.mark.parametrize("module", ["test.cli_new"]) def test_db_utils_upgrade( cli_runner, postgresql, mock_import_swhmodule, module, datadir ): """Check swh_db_upgrade""" conninfo = craft_conninfo(postgresql) result = cli_runner.invoke(swhdb, ["init-admin", module, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke(swhdb, ["init", module, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" assert swh_db_version(conninfo) == 1 new_version = swh_db_upgrade(conninfo, module) assert new_version == 6 assert swh_db_version(conninfo) == 6 versions = swh_db_versions(conninfo) # get rid of dates to ease checking versions = [(v[0], v[2]) for v in versions] assert versions[-1] == (1, "DB initialization") sqlbasedir = path.join(datadir, module.split(".", 1)[1], "sql", "upgrades") assert versions[1:-1] == [ (i, f"Upgraded to version {i} using {sqlbasedir}/{i:03d}.sql") for i in range(5, 1, -1) ] assert versions[0] == (6, "Updated version from upgrade script") cnx = BaseDb.connect(conninfo) with cnx.transaction() as cur: cur.execute("select url from origin where url like 'version%'") result = cur.fetchall() assert result == [("version%03d" % i,) for i in range(2, 7)] cur.execute( "select url from origin where url = 'this should never be executed'" ) result = cur.fetchall() assert not result @pytest.mark.parametrize("module", ["test.cli_new"]) def test_db_utils_swh_db_upgrade_sanity_checks( cli_runner, postgresql, mock_import_swhmodule, module, datadir ): """Check swh_db_upgrade""" conninfo = craft_conninfo(postgresql) result = cli_runner.invoke(swhdb, ["init-admin", module, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke(swhdb, ["init", module, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" cnx = BaseDb.connect(conninfo) with cnx.transaction() as cur: cur.execute("drop table dbmodule") # try to upgrade with a unset module with pytest.raises(ValueError): swh_db_upgrade(conninfo, module) # check the dbmodule is unset assert swh_db_module(conninfo) is None # set the stored module to something else swh_set_db_module(conninfo, f"{module}2") assert swh_db_module(conninfo) == f"{module}2" # try to upgrade with a different module with pytest.raises(ValueError): swh_db_upgrade(conninfo, module) # revert to the proper module in the db swh_set_db_module(conninfo, module, force=True) assert swh_db_module(conninfo) == module # trying again is a noop swh_set_db_module(conninfo, module) assert swh_db_module(conninfo) == module # drop the dbversion table with cnx.transaction() as cur: cur.execute("drop table dbversion") # an upgrade should fail due to missing stored version with pytest.raises(ValueError): swh_db_upgrade(conninfo, module) + + +@pytest.mark.parametrize("module", ["test.cli", "test.cli_new"]) +@pytest.mark.parametrize("flavor", [None, "default", "flavorA", "flavorB"]) +def test_db_utils_flavor(cli_runner, postgresql, mock_import_swhmodule, module, flavor): + """Check populate_database_for_package handle db flavor properly""" + conninfo = craft_conninfo(postgresql) + result = cli_runner.invoke(swhdb, ["init-admin", module, "--dbname", conninfo]) + assert result.exit_code == 0, f"Unexpected output: {result.output}" + cmd = ["init", module, "--dbname", conninfo] + if flavor: + cmd.extend(["--flavor", flavor]) + result = cli_runner.invoke(swhdb, cmd) + assert result.exit_code == 0, f"Unexpected output: {result.output}" + + # check the swh_db_module() function + assert swh_db_module(conninfo) == module + + # the dbversion and dbmodule tables exists and are populated + dbmodule, _dbversion, dbflavor = get_database_info(conninfo) + assert dbmodule == module + assert dbflavor == (flavor or "default")