diff --git a/PKG-INFO b/PKG-INFO index c46f142..92db288 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,30 +1,30 @@ Metadata-Version: 2.1 Name: swh.core -Version: 0.0.60 +Version: 0.0.61 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN -Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Source, https://forge.softwareheritage.org/source/swh-core +Project-URL: Funding, https://www.softwareheritage.org/donate Description: swh-core ======== core library for swh's modules: - config parser - hash computations - serialization - logging mechanism Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Description-Content-Type: text/markdown Provides-Extra: db -Provides-Extra: testing Provides-Extra: http +Provides-Extra: testing diff --git a/debian/changelog b/debian/changelog index 01b2c56..476af06 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,458 +1,460 @@ -swh-core (0.0.60-1~swh1~bpo9+1) stretch-swh; urgency=medium +swh-core (0.0.61-1~swh1) unstable-swh; urgency=medium - * Rebuild for stretch-swh + * New upstream release 0.0.61 - (tagged by David Douard + on 2019-05-17 10:32:07 +0200) + * Upstream changes: - v0.0.61 - -- Software Heritage autobuilder (on jenkins-debian1) Mon, 06 May 2019 13:34:02 +0000 + -- Software Heritage autobuilder (on jenkins-debian1) Fri, 17 May 2019 08:38:08 +0000 swh-core (0.0.60-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.60 - (tagged by David Douard on 2019-05-06 15:27:44 +0200) * Upstream changes: - v0.0.60 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 06 May 2019 13:32:48 +0000 swh-core (0.0.59-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.59 - (tagged by Valentin Lorentz on 2019-04-09 16:55:41 +0200) * Upstream changes: - Explicitely give Db connections back to the pool. - So they gracefully release the connection on error instead - of relying on reference-counting to call the Db's `__del__` - (which does not happen in Hypothesis tests) because a ref - to it is kept via the traceback object. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 09 Apr 2019 16:12:32 +0000 swh-core (0.0.58-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.58 - (tagged by Antoine Lambert on 2019-04-02 17:19:05 +0200) * Upstream changes: - version 0.0.58 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 02 Apr 2019 15:24:34 +0000 swh-core (0.0.57-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.57 - (tagged by Nicolas Dandrimont on 2019-03-28 15:51:27 +0100) * Upstream changes: - Release swh.core v0.0.57 - Move to native async primitives - Fix statsd.timed exceptional behavior bug/misfeature - Fix SWHRemoteAPI post_stream method -- Software Heritage autobuilder (on jenkins-debian1) Thu, 28 Mar 2019 14:55:58 +0000 swh-core (0.0.56-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.56 - (tagged by David Douard on 2019-03-19 10:17:06 +0100) * Upstream changes: - v0.0.56 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 19 Mar 2019 09:27:18 +0000 swh-core (0.0.55-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.55 - (tagged by Antoine R. Dumont (@ardumont) on 2019-02-19 12:28:26 +0100) * Upstream changes: - v0.0.55 - Fix runtime dependencies -- Software Heritage autobuilder (on jenkins-debian1) Tue, 19 Feb 2019 11:32:28 +0000 swh-core (0.0.54-1~swh2) unstable-swh; urgency=medium * New upstream release 0.0.54 * Upstream changes: - Add missing build dependencies -- Antoine R. Dumont (@ardumont) Tue, 12 Feb 2019 16:25:34 +0000 swh-core (0.0.54-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.54 - (tagged by Valentin Lorentz on 2019-02-11 16:47:18 +0100) * Upstream changes: - Add test for BaseDb.connect. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 12 Feb 2019 12:37:43 +0000 swh-core (0.0.53-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.53 - (tagged by Antoine R. Dumont (@ardumont) on 2019-02-08 09:09:30 +0100) * Upstream changes: - v0.0.53 - Fix debian build -- Software Heritage autobuilder (on jenkins-debian1) Fri, 08 Feb 2019 08:12:31 +0000 swh-core (0.0.52-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.52 - (tagged by David Douard on 2019-02-06 15:24:04 +0100) * Upstream changes: - v0.0.52 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 06 Feb 2019 14:27:14 +0000 swh-core (0.0.51-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.51 - (tagged by David Douard on 2019-02-01 14:28:27 +0100) * Upstream changes: - v0.0.51 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 01 Feb 2019 13:31:45 +0000 swh-core (0.0.50-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.50 - (tagged by Nicolas Dandrimont on 2019-01-09 15:50:58 +0100) * Upstream changes: - Release swh.core v0.0.50 - Add statsd client module - Log used config files -- Software Heritage autobuilder (on jenkins-debian1) Wed, 09 Jan 2019 14:54:37 +0000 swh-core (0.0.49-1~swh1) unstable-swh; urgency=medium * Make DbTestFixture.setUp() accept and pass *args and **kwargs. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 08 Jan 2019 16:38:02 +0000 swh-core (0.0.48-1~swh1) unstable-swh; urgency=medium * v0.0.48 * swh.core.cli: Update swh-db-init to make it idemtpotent -- Antoine R. Dumont (@ardumont) Tue, 08 Jan 2019 15:33:15 +0000 swh-core (0.0.47-1~swh1) unstable-swh; urgency=medium * v0.0.47 * swh.core.cli: Fix flag -- Antoine R. Dumont (@ardumont) Tue, 08 Jan 2019 15:16:09 +0000 swh-core (0.0.46-1~swh1) unstable-swh; urgency=medium * v0.0.46 * utils.grouper: Improve implementation * Remove now-obsolete information about swh.core.worker -- Antoine R. Dumont (@ardumont) Tue, 08 Jan 2019 14:37:34 +0000 swh-core (0.0.45-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.45 * Compatibility with recent msgpack * Debian packaging-related cleanups -- Nicolas Dandrimont Thu, 22 Nov 2018 21:09:53 +0100 swh-core (0.0.44-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.44 * Refactor the database testing fixtures * Stop unsafe serialization/deserialization constructs * Update tests to use nose -- Nicolas Dandrimont Thu, 18 Oct 2018 18:20:12 +0200 swh-core (0.0.43-1~swh1) unstable-swh; urgency=medium * v0.0.43 * Fix missing dependency declaration -- Antoine R. Dumont (@ardumont) Thu, 11 Oct 2018 15:47:06 +0200 swh-core (0.0.42-1~swh1) unstable-swh; urgency=medium * v0.0.42 * Fix missing dependency declaration -- Antoine R. Dumont (@ardumont) Thu, 11 Oct 2018 15:45:25 +0200 swh-core (0.0.41-1~swh1) unstable-swh; urgency=medium * Add functions to generate HTTP API clients and servers from databases. * Summary: This moves the interesting parts of D505 into the core, so other components can use them as well. * Test Plan: `make test` * Reviewers: ardumont, seirl, #reviewers * Reviewed By: ardumont, #reviewers * Subscribers: douardda * Differential Revision: https://forge.softwareheritage.org/D507 -- Valentin Lorentz Thu, 11 Oct 2018 10:57:27 +0200 swh-core (0.0.40-1~swh1) unstable-swh; urgency=medium * v0.0.40 * swh.core.api.SWHRemoteAPI: Permit to set a query timeout option -- Antoine R. Dumont (@ardumont) Thu, 24 May 2018 12:10:03 +0200 swh-core (0.0.39-1~swh1) unstable-swh; urgency=medium * v0.0.39 * package: Add missing runtime dependency -- Antoine R. Dumont (@ardumont) Thu, 26 Apr 2018 15:24:22 +0200 swh-core (0.0.38-1~swh1) unstable-swh; urgency=medium * v0.0.38 * tests: Use more reasonable psql options for db restores * swh.core.serializers: Add custom types serialization -- Antoine R. Dumont (@ardumont) Thu, 26 Apr 2018 15:15:27 +0200 swh-core (0.0.37-1~swh1) unstable-swh; urgency=medium * v0.0.37 * Move test fixture in swh.core.tests.server_testing module -- Antoine R. Dumont (@ardumont) Wed, 25 Apr 2018 15:00:02 +0200 swh-core (0.0.36-1~swh1) unstable-swh; urgency=medium * v0.0.36 * Migrate swh.loader.tar.tarball module in swh.core -- Antoine R. Dumont (@ardumont) Wed, 06 Dec 2017 12:03:29 +0100 swh-core (0.0.35-1~swh1) unstable-swh; urgency=medium * Release swh.core version 0.0.35 * Update packaging runes -- Nicolas Dandrimont Thu, 12 Oct 2017 18:07:50 +0200 swh-core (0.0.34-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.34 * New modular database test fixture -- Nicolas Dandrimont Mon, 07 Aug 2017 18:29:48 +0200 swh-core (0.0.33-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.33 * Be more conservative with remote API responses -- Nicolas Dandrimont Mon, 19 Jun 2017 19:01:38 +0200 swh-core (0.0.32-1~swh1) unstable-swh; urgency=medium * Release swh-core v0.0.32 * Add asynchronous streaming methods for internal APIs * Remove task arguments from systemd-journal loggers -- Nicolas Dandrimont Tue, 09 May 2017 14:04:22 +0200 swh-core (0.0.31-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.31 * Add explicit dependency on python3-systemd -- Nicolas Dandrimont Fri, 07 Apr 2017 15:11:26 +0200 swh-core (0.0.30-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.30 * drop swh.core.hashutil (moved to swh.model.hashutil) * add a systemd logger -- Nicolas Dandrimont Fri, 07 Apr 2017 11:49:15 +0200 swh-core (0.0.29-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.29 * Catch proper exception in the base API client -- Nicolas Dandrimont Thu, 02 Feb 2017 00:19:25 +0100 swh-core (0.0.28-1~swh1) unstable-swh; urgency=medium * v0.0.28 * Refactoring some common code into swh.core -- Antoine R. Dumont (@ardumont) Thu, 26 Jan 2017 14:54:22 +0100 swh-core (0.0.27-1~swh1) unstable-swh; urgency=medium * v0.0.27 * Fix issue with default boolean value -- Antoine R. Dumont (@ardumont) Thu, 20 Oct 2016 16:15:20 +0200 swh-core (0.0.26-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.26 * Raise an exception when a configuration file exists and is unreadable -- Nicolas Dandrimont Wed, 12 Oct 2016 10:16:09 +0200 swh-core (0.0.25-1~swh1) unstable-swh; urgency=medium * v0.0.25 * Add new function utils.cwd -- Antoine R. Dumont (@ardumont) Thu, 29 Sep 2016 21:29:37 +0200 swh-core (0.0.24-1~swh1) unstable-swh; urgency=medium * v0.0.24 * Deal with edge case in logger regarding json -- Antoine R. Dumont (@ardumont) Thu, 22 Sep 2016 12:21:09 +0200 swh-core (0.0.23-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.23 * Properly fix the PyYAML dependency -- Nicolas Dandrimont Tue, 23 Aug 2016 16:20:29 +0200 swh-core (0.0.22-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.22 * Proper loading of yaml and ini files in all paths -- Nicolas Dandrimont Fri, 19 Aug 2016 15:45:55 +0200 swh-core (0.0.21-1~swh1) unstable-swh; urgency=medium * v0.0.21 * Update test tools -- Antoine R. Dumont (@ardumont) Tue, 19 Jul 2016 14:47:01 +0200 swh-core (0.0.20-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.20 * Add some generic bytes <-> escaped unicode methods -- Nicolas Dandrimont Tue, 14 Jun 2016 16:54:41 +0200 swh-core (0.0.19-1~swh1) unstable-swh; urgency=medium * v0.0.19 * Resurrect swh.core.utils -- Antoine R. Dumont (@ardumont) Fri, 15 Apr 2016 12:40:43 +0200 swh-core (0.0.18-1~swh1) unstable-swh; urgency=medium * v0.0.18 * Add swh.core.utils * serializers: support UUIDs all around -- Antoine R. Dumont (@ardumont) Sat, 26 Mar 2016 11:16:33 +0100 swh-core (0.0.17-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.17 * Allow serialization of UUIDs -- Nicolas Dandrimont Fri, 04 Mar 2016 11:40:56 +0100 swh-core (0.0.16-1~swh1) unstable-swh; urgency=medium * Release swh.core version 0.0.16 * add bytehex_to_hash and hash_to_bytehex in hashutil * move scheduling utilities to swh.scheduler -- Nicolas Dandrimont Fri, 19 Feb 2016 18:12:10 +0100 swh-core (0.0.15-1~swh1) unstable-swh; urgency=medium * Release v0.0.15 * Add hashutil.hash_git_object -- Nicolas Dandrimont Wed, 16 Dec 2015 16:31:26 +0100 swh-core (0.0.14-1~swh1) unstable-swh; urgency=medium * v0.0.14 * Add simple README * Update license * swh.core.hashutil.hashfile can now deal with filepath as bytes -- Antoine R. Dumont (@ardumont) Fri, 23 Oct 2015 11:13:14 +0200 swh-core (0.0.13-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.13 -- Nicolas Dandrimont Fri, 09 Oct 2015 17:32:49 +0200 swh-core (0.0.12-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.12 -- Nicolas Dandrimont Tue, 06 Oct 2015 17:34:34 +0200 swh-core (0.0.11-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.11 -- Nicolas Dandrimont Sat, 03 Oct 2015 15:57:03 +0200 swh-core (0.0.10-1~swh1) unstable-swh; urgency=medium * Prepare deploying swh.core v0.0.10 -- Nicolas Dandrimont Sat, 03 Oct 2015 12:28:52 +0200 swh-core (0.0.9-1~swh1) unstable-swh; urgency=medium * Prepare deploying swh.core v0.0.9 -- Nicolas Dandrimont Sat, 03 Oct 2015 11:36:55 +0200 swh-core (0.0.8-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.8 -- Nicolas Dandrimont Thu, 01 Oct 2015 12:31:44 +0200 swh-core (0.0.7-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.7 -- Nicolas Dandrimont Thu, 01 Oct 2015 11:29:04 +0200 swh-core (0.0.6-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.6 -- Nicolas Dandrimont Tue, 29 Sep 2015 16:48:44 +0200 swh-core (0.0.5-1~swh1) unstable-swh; urgency=medium * Prepare v0.0.5 deployment -- Nicolas Dandrimont Tue, 29 Sep 2015 16:08:32 +0200 swh-core (0.0.4-1~swh1) unstable-swh; urgency=medium * Tagging swh.core 0.0.4 -- Nicolas Dandrimont Fri, 25 Sep 2015 15:41:26 +0200 swh-core (0.0.3-1~swh1) unstable-swh; urgency=medium * Tag swh.core v0.0.3 -- Nicolas Dandrimont Fri, 25 Sep 2015 11:07:10 +0200 swh-core (0.0.2-1~swh1) unstable-swh; urgency=medium * Deploy v0.0.2 -- Nicolas Dandrimont Wed, 23 Sep 2015 12:08:50 +0200 swh-core (0.0.1-1~swh1) unstable-swh; urgency=medium * Initial release * Tag v0.0.1 for deployment -- Nicolas Dandrimont Tue, 22 Sep 2015 14:52:26 +0200 diff --git a/requirements.txt b/requirements.txt index cd36a30..bd236c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,3 @@ PyYAML systemd-python -# these deps below are now handled in dedicated 'extras' and should be removed -# from this main requirement file ASAP -arrow -aiohttp -msgpack > 0.5 -psycopg2 -python-dateutil -requests -Flask -decorator diff --git a/swh.core.egg-info/PKG-INFO b/swh.core.egg-info/PKG-INFO index c46f142..92db288 100644 --- a/swh.core.egg-info/PKG-INFO +++ b/swh.core.egg-info/PKG-INFO @@ -1,30 +1,30 @@ Metadata-Version: 2.1 Name: swh.core -Version: 0.0.60 +Version: 0.0.61 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN -Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Source, https://forge.softwareheritage.org/source/swh-core +Project-URL: Funding, https://www.softwareheritage.org/donate Description: swh-core ======== core library for swh's modules: - config parser - hash computations - serialization - logging mechanism Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Description-Content-Type: text/markdown Provides-Extra: db -Provides-Extra: testing Provides-Extra: http +Provides-Extra: testing diff --git a/swh.core.egg-info/SOURCES.txt b/swh.core.egg-info/SOURCES.txt index 8590f71..32148b1 100644 --- a/swh.core.egg-info/SOURCES.txt +++ b/swh.core.egg-info/SOURCES.txt @@ -1,41 +1,44 @@ MANIFEST.in Makefile README.md requirements-swh.txt requirements.txt setup.py version.txt swh/__init__.py swh.core.egg-info/PKG-INFO swh.core.egg-info/SOURCES.txt swh.core.egg-info/dependency_links.txt swh.core.egg-info/entry_points.txt swh.core.egg-info/requires.txt swh.core.egg-info/top_level.txt swh/core/__init__.py swh/core/api_async.py swh/core/config.py swh/core/logger.py swh/core/statsd.py swh/core/tarball.py swh/core/utils.py swh/core/api/__init__.py swh/core/api/asynchronous.py swh/core/api/negotiation.py swh/core/api/serializers.py +swh/core/api/tests/__init__.py +swh/core/api/tests/server_testing.py +swh/core/api/tests/test_api.py +swh/core/api/tests/test_serializers.py swh/core/cli/__init__.py swh/core/cli/db.py swh/core/db/__init__.py swh/core/db/common.py swh/core/db/db_utils.py +swh/core/db/tests/__init__.py +swh/core/db/tests/conftest.py +swh/core/db/tests/db_testing.py +swh/core/db/tests/test_db.py swh/core/sql/log-schema.sql swh/core/tests/__init__.py -swh/core/tests/conftest.py -swh/core/tests/db_testing.py -swh/core/tests/server_testing.py -swh/core/tests/test_api.py +swh/core/tests/test_cli.py swh/core/tests/test_config.py -swh/core/tests/test_db.py -swh/core/tests/test_serializers.py swh/core/tests/test_statsd.py swh/core/tests/test_utils.py \ No newline at end of file diff --git a/swh.core.egg-info/requires.txt b/swh.core.egg-info/requires.txt index 1614931..131c64f 100644 --- a/swh.core.egg-info/requires.txt +++ b/swh.core.egg-info/requires.txt @@ -1,36 +1,29 @@ PyYAML systemd-python -arrow -aiohttp -msgpack>0.5 -psycopg2 -python-dateutil -requests -Flask -decorator [db] psycopg2 [http] aiohttp arrow decorator Flask msgpack>0.5 python-dateutil requests [testing] +Click pytest<4 pytest-postgresql requests-mock hypothesis>=3.11.0 psycopg2 aiohttp arrow decorator Flask msgpack>0.5 python-dateutil requests diff --git a/swh/core/api/__init__.py b/swh/core/api/__init__.py index 5c48358..0a23f66 100644 --- a/swh/core/api/__init__.py +++ b/swh/core/api/__init__.py @@ -1,322 +1,322 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import collections import functools import inspect import json import logging import pickle import requests import datetime from flask import Flask, Request, Response, request, abort from .serializers import (decode_response, encode_data_client as encode_data, msgpack_dumps, msgpack_loads, SWHJSONDecoder) from .negotiation import (Formatter as FormatterBase, Negotiator as NegotiatorBase, negotiate as _negotiate) logger = logging.getLogger(__name__) -# support for content negotation +# support for content negotiation class Negotiator(NegotiatorBase): def best_mimetype(self): return request.accept_mimetypes.best_match( self.accept_mimetypes, 'application/json') def _abort(self, status_code, err=None): return abort(status_code, err) def negotiate(formatter_cls, *args, **kwargs): return _negotiate(Negotiator, formatter_cls, *args, **kwargs) class Formatter(FormatterBase): def _make_response(self, body, content_type): return Response(body, content_type=content_type) class SWHJSONEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, (datetime.datetime, datetime.date)): return obj.isoformat() if isinstance(obj, datetime.timedelta): return str(obj) # Let the base class default method raise the TypeError return super().default(obj) class JSONFormatter(Formatter): format = 'json' mimetypes = ['application/json'] def render(self, obj): return json.dumps(obj, cls=SWHJSONEncoder) class MsgpackFormatter(Formatter): format = 'msgpack' mimetypes = ['application/x-msgpack'] def render(self, obj): return msgpack_dumps(obj) # base API classes class RemoteException(Exception): pass def remote_api_endpoint(path): def dec(f): f._endpoint_path = path return f return dec class APIError(Exception): """API Error""" def __str__(self): return ('An unexpected error occurred in the backend: {}' .format(self.args)) class MetaSWHRemoteAPI(type): """Metaclass for SWHRemoteAPI, which adds a method for each endpoint of the database it is designed to access. See for example :class:`swh.indexer.storage.api.client.RemoteStorage`""" def __new__(cls, name, bases, attributes): # For each method wrapped with @remote_api_endpoint in an API backend # (eg. :class:`swh.indexer.storage.IndexerStorage`), add a new # method in RemoteStorage, with the same documentation. # # Note that, despite the usage of decorator magic (eg. functools.wrap), # this never actually calls an IndexerStorage method. backend_class = attributes.get('backend_class', None) for base in bases: if backend_class is not None: break backend_class = getattr(base, 'backend_class', None) if backend_class: for (meth_name, meth) in backend_class.__dict__.items(): if hasattr(meth, '_endpoint_path'): cls.__add_endpoint(meth_name, meth, attributes) return super().__new__(cls, name, bases, attributes) @staticmethod def __add_endpoint(meth_name, meth, attributes): wrapped_meth = inspect.unwrap(meth) @functools.wraps(meth) # Copy signature and doc def meth_(*args, **kwargs): # Match arguments and parameters post_data = inspect.getcallargs( wrapped_meth, *args, **kwargs) # Remove arguments that should not be passed self = post_data.pop('self') post_data.pop('cur', None) post_data.pop('db', None) # Send the request. return self.post(meth._endpoint_path, post_data) attributes[meth_name] = meth_ class SWHRemoteAPI(metaclass=MetaSWHRemoteAPI): """Proxy to an internal SWH API """ backend_class = None """For each method of `backend_class` decorated with :func:`remote_api_endpoint`, a method with the same prototype and docstring will be added to this class. Calls to this new method will be translated into HTTP requests to a remote server. This backend class will never be instantiated, it only serves as a template.""" api_exception = APIError """The exception class to raise in case of communication error with the server.""" def __init__(self, url, api_exception=None, timeout=None, chunk_size=4096, **kwargs): if api_exception: self.api_exception = api_exception base_url = url if url.endswith('/') else url + '/' self.url = base_url self.session = requests.Session() self.timeout = timeout self.chunk_size = chunk_size def _url(self, endpoint): return '%s%s' % (self.url, endpoint) def raw_verb(self, verb, endpoint, **opts): if 'chunk_size' in opts: # if the chunk_size argument has been passed, consider the user # also wants stream=True, otherwise, what's the point. opts['stream'] = True if self.timeout and 'timeout' not in opts: opts['timeout'] = self.timeout try: return getattr(self.session, verb)( self._url(endpoint), **opts ) except requests.exceptions.ConnectionError as e: raise self.api_exception(e) def post(self, endpoint, data, **opts): if isinstance(data, (collections.Iterator, collections.Generator)): data = (encode_data(x) for x in data) else: data = encode_data(data) chunk_size = opts.pop('chunk_size', self.chunk_size) response = self.raw_verb( 'post', endpoint, data=data, headers={'content-type': 'application/x-msgpack', 'accept': 'application/x-msgpack'}, **opts) if opts.get('stream') or \ response.headers.get('transfer-encoding') == 'chunked': return response.iter_content(chunk_size) else: return self._decode_response(response) post_stream = post def get(self, endpoint, **opts): chunk_size = opts.pop('chunk_size', self.chunk_size) response = self.raw_verb( 'get', endpoint, headers={'accept': 'application/x-msgpack'}, **opts) if opts.get('stream') or \ response.headers.get('transfer-encoding') == 'chunked': return response.iter_content(chunk_size) else: return self._decode_response(response) def get_stream(self, endpoint, **opts): return self.get(endpoint, stream=True, **opts) def _decode_response(self, response): if response.status_code == 404: return None if response.status_code == 500: data = decode_response(response) if 'exception_pickled' in data: raise pickle.loads(data['exception_pickled']) else: raise RemoteException(data['exception']) # XXX: this breaks language-independence and should be # replaced by proper unserialization if response.status_code == 400: raise pickle.loads(decode_response(response)) elif response.status_code != 200: raise RemoteException( "Unexpected status code for API request: %s (%s)" % ( response.status_code, response.content, ) ) return decode_response(response) def __repr__(self): return '<{} url={}>'.format(self.__class__.__name__, self.url) class BytesRequest(Request): """Request with proper escaping of arbitrary byte sequences.""" encoding = 'utf-8' encoding_errors = 'surrogateescape' ENCODERS = { 'application/x-msgpack': msgpack_dumps, 'application/json': json.dumps, } def encode_data_server(data, content_type='application/x-msgpack'): encoded_data = ENCODERS[content_type](data) return Response( encoded_data, mimetype=content_type, ) def decode_request(request): content_type = request.mimetype data = request.get_data() if not data: return {} if content_type == 'application/x-msgpack': r = msgpack_loads(data) elif content_type == 'application/json': r = json.loads(data, cls=SWHJSONDecoder) else: raise ValueError('Wrong content type `%s` for API request' % content_type) return r def error_handler(exception, encoder): # XXX: this breaks language-independence and should be # replaced by proper serialization of errors logging.exception(exception) response = encoder(pickle.dumps(exception)) response.status_code = 400 return response class SWHServerAPIApp(Flask): """For each endpoint of the given `backend_class`, tells app.route to call a function that decodes the request and sends it to the backend object provided by the factory. :param Any backend_class: The class of the backend, which will be analyzed to look for API endpoints. :param Callable[[], backend_class] backend_factory: A function with no argument that returns an instance of `backend_class`.""" request_class = BytesRequest def __init__(self, *args, backend_class=None, backend_factory=None, **kwargs): super().__init__(*args, **kwargs) if backend_class is not None: if backend_factory is None: raise TypeError('Missing argument backend_factory') for (meth_name, meth) in backend_class.__dict__.items(): if hasattr(meth, '_endpoint_path'): self.__add_endpoint(meth_name, meth, backend_factory) def __add_endpoint(self, meth_name, meth, backend_factory): from flask import request @self.route('/'+meth._endpoint_path, methods=['POST']) @functools.wraps(meth) # Copy signature and doc def _f(): # Call the actual code obj_meth = getattr(backend_factory(), meth_name) return encode_data_server(obj_meth(**decode_request(request))) diff --git a/swh/core/api/tests/__init__.py b/swh/core/api/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/swh/core/tests/server_testing.py b/swh/core/api/tests/server_testing.py similarity index 100% rename from swh/core/tests/server_testing.py rename to swh/core/api/tests/server_testing.py diff --git a/swh/core/tests/test_api.py b/swh/core/api/tests/test_api.py similarity index 100% rename from swh/core/tests/test_api.py rename to swh/core/api/tests/test_api.py diff --git a/swh/core/tests/test_serializers.py b/swh/core/api/tests/test_serializers.py similarity index 100% rename from swh/core/tests/test_serializers.py rename to swh/core/api/tests/test_serializers.py diff --git a/swh/core/cli/__init__.py b/swh/core/cli/__init__.py index de82173..eefac67 100644 --- a/swh/core/cli/__init__.py +++ b/swh/core/cli/__init__.py @@ -1,40 +1,60 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import click import logging import pkg_resources +LOG_LEVEL_NAMES = ['NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] + +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) + logger = logging.getLogger(__name__) -CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) +class AliasedGroup(click.Group): + 'A simple Group that supports command aliases' + + @property + def aliases(self): + if not hasattr(self, '_aliases'): + self._aliases = {} + return self._aliases + + def get_command(self, ctx, cmd_name): + return super().get_command(ctx, self.aliases.get(cmd_name, cmd_name)) + + def add_alias(self, name, alias): + if not isinstance(name, str): + name = name.name + self.aliases[alias] = name -@click.group(context_settings=CONTEXT_SETTINGS) +@click.group(context_settings=CONTEXT_SETTINGS, cls=AliasedGroup) @click.option('--log-level', '-l', default='INFO', - type=click.Choice(logging._nameToLevel.keys()), + type=click.Choice(LOG_LEVEL_NAMES), help="Log level (default to INFO)") @click.pass_context def swh(ctx, log_level): - """Software Heritage Tool + """Command line interface for Software Heritage. """ - logger.setLevel(log_level) + log_level = logging.getLevelName(log_level) + logging.root.setLevel(log_level) ctx.ensure_object(dict) - ctx.obj['log_level'] = logging._nameToLevel[log_level] + ctx.obj['log_level'] = log_level def main(): logging.basicConfig() # load plugins that define cli sub commands for entry_point in pkg_resources.iter_entry_points('swh.cli.subcommands'): cmd = entry_point.load() swh.add_command(cmd, name=entry_point.name) return swh(auto_envvar_prefix='SWH') if __name__ == '__main__': main() diff --git a/swh/core/cli/db.py b/swh/core/cli/db.py index 27a1992..a258ea7 100755 --- a/swh/core/cli/db.py +++ b/swh/core/cli/db.py @@ -1,86 +1,87 @@ #!/usr/bin/env python3 # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import warnings warnings.filterwarnings("ignore") # noqa prevent psycopg from telling us sh*t import click +from swh.core.cli import CONTEXT_SETTINGS logger = logging.getLogger(__name__) -@click.command() +@click.command(context_settings=CONTEXT_SETTINGS) @click.argument('module', nargs=-1, required=True) @click.option('--db-name', '-d', help='Database name.', default='softwareheritage-dev', show_default=True) def db_init(module, db_name=None): """Initialise a database for the Software Heritage . By default, attempts to create the database first. Example: swh db-init -d swh-test storage If you want to specify non-default postgresql connection parameters, please provide them using standard environment variables. See psql(1) man page (section ENVIRONMENTS) for details. Example: PGPORT=5434 swh db-init indexer """ # put import statements here so we can keep startup time of the main swh # command as short as possible from os import path import glob from importlib import import_module from swh.core.utils import numfile_sortkey as sortkey from swh.core.tests.db_testing import ( pg_createdb, pg_restore, DB_DUMP_TYPES, swh_db_version ) logger.debug('db_init %s dn_name=%s', module, db_name) dump_files = [] for modname in module: if not modname.startswith('swh.'): modname = 'swh.{}'.format(modname) try: m = import_module(modname) except ImportError: raise click.BadParameter( 'Unable to load module {}'.format(modname)) sqldir = path.join(path.dirname(m.__file__), 'sql') if not path.isdir(sqldir): raise click.BadParameter( 'Module {} does not provide a db schema ' '(no sql/ dir)'.format(modname)) dump_files.extend(sorted(glob.glob(path.join(sqldir, '*.sql')), key=sortkey)) # Create the db (or fail silently if already existing) pg_createdb(db_name, check=False) # Try to retrieve the db version if any db_version = swh_db_version(db_name) if not db_version: # Initialize the db dump_files = [(x, DB_DUMP_TYPES[path.splitext(x)[1]]) for x in dump_files] for dump, dtype in dump_files: click.secho('Loading {}'.format(dump), fg='yellow') pg_restore(db_name, dump, dtype) db_version = swh_db_version(db_name) # TODO: Ideally migrate the version from db_version to the latest # db version click.secho('DONE database is {} version {}'.format(db_name, db_version), fg='green', bold=True) diff --git a/swh/core/db/__init__.py b/swh/core/db/__init__.py index da0e402..455f56d 100644 --- a/swh/core/db/__init__.py +++ b/swh/core/db/__init__.py @@ -1,193 +1,193 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import binascii import datetime import enum import json import os import threading from contextlib import contextmanager import psycopg2 import psycopg2.extras psycopg2.extras.register_uuid() def escape(data): if data is None: return '' if isinstance(data, bytes): return '\\x%s' % binascii.hexlify(data).decode('ascii') elif isinstance(data, str): return '"%s"' % data.replace('"', '""') elif isinstance(data, datetime.datetime): # We escape twice to make sure the string generated by # isoformat gets escaped return escape(data.isoformat()) elif isinstance(data, dict): return escape(json.dumps(data)) elif isinstance(data, list): return escape("{%s}" % ','.join(escape(d) for d in data)) elif isinstance(data, psycopg2.extras.Range): # We escape twice here too, so that we make sure # everything gets passed to copy properly return escape( '%s%s,%s%s' % ( '[' if data.lower_inc else '(', '-infinity' if data.lower_inf else escape(data.lower), 'infinity' if data.upper_inf else escape(data.upper), ']' if data.upper_inc else ')', ) ) elif isinstance(data, enum.IntEnum): return escape(int(data)) else: # We don't escape here to make sure we pass literals properly return str(data) def typecast_bytea(value, cur): if value is not None: data = psycopg2.BINARY(value, cur) return data.tobytes() class BaseDb: """Base class for swh.*.*Db. cf. swh.storage.db.Db, swh.archiver.db.ArchiverDb """ @classmethod def adapt_conn(cls, conn): """Makes psycopg2 use 'bytes' to decode bytea instead of 'memoryview', for this connection.""" cur = conn.cursor() cur.execute("SELECT null::bytea, null::bytea[]") bytea_oid = cur.description[0][1] bytea_array_oid = cur.description[1][1] t_bytes = psycopg2.extensions.new_type( (bytea_oid,), "bytea", typecast_bytea) psycopg2.extensions.register_type(t_bytes, conn) t_bytes_array = psycopg2.extensions.new_array_type( (bytea_array_oid,), "bytea[]", t_bytes) psycopg2.extensions.register_type(t_bytes_array, conn) @classmethod def connect(cls, *args, **kwargs): """factory method to create a DB proxy Accepts all arguments of psycopg2.connect; only some specific possibilities are reported below. Args: connstring: libpq2 connection string """ conn = psycopg2.connect(*args, **kwargs) return cls(conn) @classmethod def from_pool(cls, pool): conn = pool.getconn() return cls(conn, pool=pool) def __init__(self, conn, pool=None): """create a DB proxy Args: conn: psycopg2 connection to the SWH DB pool: psycopg2 pool of connections """ self.adapt_conn(conn) self.conn = conn self.pool = pool def put_conn(self): if self.pool: self.pool.putconn(self.conn) def cursor(self, cur_arg=None): """get a cursor: from cur_arg if given, or a fresh one otherwise meant to avoid boilerplate if/then/else in methods that proxy stored procedures """ if cur_arg is not None: return cur_arg else: return self.conn.cursor() _cursor = cursor # for bw compat @contextmanager def transaction(self): """context manager to execute within a DB transaction Yields: a psycopg2 cursor """ with self.conn.cursor() as cur: try: yield cur self.conn.commit() except Exception: if not self.conn.closed: self.conn.rollback() raise def copy_to(self, items, tblname, columns, cur=None, item_cb=None, default_values={}): """Copy items' entries to table tblname with columns information. Args: items (List[dict]): dictionaries of data to copy over tblname. tblname (str): destination table's name. columns ([str]): keys to access data in items and also the column names in the destination table. - default_values (dict): dictionnary of default values to use when + default_values (dict): dictionary of default values to use when inserting entried int the tblname table. cur: a db cursor; if not given, a new cursor will be created. item_cb (fn): optional function to apply to items's entry. """ read_file, write_file = os.pipe() def writer(): cursor = self.cursor(cur) with open(read_file, 'r') as f: cursor.copy_expert('COPY %s (%s) FROM STDIN CSV' % ( tblname, ', '.join(columns)), f) write_thread = threading.Thread(target=writer) write_thread.start() try: with open(write_file, 'w') as f: for d in items: if item_cb is not None: item_cb(d) line = [escape(d.get(k, default_values.get(k))) for k in columns] f.write(','.join(line)) f.write('\n') finally: # No problem bubbling up exceptions, but we still need to make sure # we finish copying, even though we're probably going to cancel the # transaction. write_thread.join() def mktemp(self, tblname, cur=None): self.cursor(cur).execute('SELECT swh_mktemp(%s)', (tblname,)) diff --git a/swh/core/db/tests/__init__.py b/swh/core/db/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/swh/core/tests/conftest.py b/swh/core/db/tests/conftest.py similarity index 100% rename from swh/core/tests/conftest.py rename to swh/core/db/tests/conftest.py diff --git a/swh/core/tests/db_testing.py b/swh/core/db/tests/db_testing.py similarity index 100% rename from swh/core/tests/db_testing.py rename to swh/core/db/tests/db_testing.py diff --git a/swh/core/tests/test_db.py b/swh/core/db/tests/test_db.py similarity index 98% rename from swh/core/tests/test_db.py rename to swh/core/db/tests/test_db.py index a9c70e8..355a384 100644 --- a/swh/core/tests/test_db.py +++ b/swh/core/db/tests/test_db.py @@ -1,102 +1,102 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os.path import tempfile import unittest from hypothesis import strategies, given import pytest from swh.core.db import BaseDb -from swh.core.tests.db_testing import ( +from .db_testing import ( SingleDbTestFixture, db_create, db_destroy, db_close, ) INIT_SQL = ''' create table test_table ( i int, txt text, bytes bytea ); ''' db_rows = strategies.lists(strategies.tuples( strategies.integers(-2147483648, +2147483647), strategies.text( alphabet=strategies.characters( blacklist_categories=['Cs'], # surrogates blacklist_characters=[ '\x00', # pgsql does not support the null codepoint '\r', # pgsql normalizes those ] ), ), strategies.binary(), )) @pytest.mark.db def test_connect(): db_name = db_create('test-db2', dumps=[]) try: db = BaseDb.connect('dbname=%s' % db_name) with db.cursor() as cur: cur.execute(INIT_SQL) cur.execute("insert into test_table values (1, %s, %s);", ('foo', b'bar')) cur.execute("select * from test_table;") assert list(cur) == [(1, 'foo', b'bar')] finally: db_close(db.conn) db_destroy(db_name) @pytest.mark.db class TestDb(SingleDbTestFixture, unittest.TestCase): TEST_DB_NAME = 'test-db' @classmethod def setUpClass(cls): with tempfile.TemporaryDirectory() as td: with open(os.path.join(td, 'init.sql'), 'a') as fd: fd.write(INIT_SQL) cls.TEST_DB_DUMP = os.path.join(td, '*.sql') super().setUpClass() def setUp(self): super().setUp() self.db = BaseDb(self.conn) def test_initialized(self): cur = self.db.cursor() cur.execute("insert into test_table values (1, %s, %s);", ('foo', b'bar')) cur.execute("select * from test_table;") self.assertEqual(list(cur), [(1, 'foo', b'bar')]) def test_reset_tables(self): cur = self.db.cursor() cur.execute("insert into test_table values (1, %s, %s);", ('foo', b'bar')) self.reset_db_tables('test-db') cur.execute("select * from test_table;") self.assertEqual(list(cur), []) @given(db_rows) def test_copy_to(self, data): # the table is not reset between runs by hypothesis self.reset_db_tables('test-db') items = [dict(zip(['i', 'txt', 'bytes'], item)) for item in data] self.db.copy_to(items, 'test_table', ['i', 'txt', 'bytes']) cur = self.db.cursor() cur.execute('select * from test_table;') self.assertCountEqual(list(cur), data) diff --git a/swh/core/tests/test_cli.py b/swh/core/tests/test_cli.py new file mode 100644 index 0000000..63f2538 --- /dev/null +++ b/swh/core/tests/test_cli.py @@ -0,0 +1,109 @@ +# + +import logging + +import click +from click.testing import CliRunner + +from swh.core.cli import swh as swhmain + + +help_msg = '''Usage: swh [OPTIONS] COMMAND [ARGS]... + + Command line interface for Software Heritage. + +Options: + -l, --log-level [NOTSET|DEBUG|INFO|WARNING|ERROR|CRITICAL] + Log level (default to INFO) + -h, --help Show this message and exit. +''' + + +def test_swh_help(): + runner = CliRunner() + result = runner.invoke(swhmain, ['-h']) + assert result.exit_code == 0 + assert result.output == help_msg + + result = runner.invoke(swhmain, ['--help']) + assert result.exit_code == 0 + assert result.output == help_msg + + +def test_command(): + @swhmain.command(name='test') + @click.pass_context + def swhtest(ctx): + click.echo('Hello SWH!') + + runner = CliRunner() + result = runner.invoke(swhmain, ['test']) + assert result.exit_code == 0 + assert result.output.strip() == 'Hello SWH!' + + +def test_loglevel_default(caplog): + @swhmain.command(name='test') + @click.pass_context + def swhtest(ctx): + assert logging.root.level == 20 + click.echo('Hello SWH!') + + runner = CliRunner() + result = runner.invoke(swhmain, ['test']) + assert result.exit_code == 0 + print(result.output) + assert result.output.strip() == '''Hello SWH!''' + + +def test_loglevel_error(caplog): + @swhmain.command(name='test') + @click.pass_context + def swhtest(ctx): + assert logging.root.level == 40 + click.echo('Hello SWH!') + + runner = CliRunner() + result = runner.invoke(swhmain, ['-l', 'ERROR', 'test']) + assert result.exit_code == 0 + assert result.output.strip() == '''Hello SWH!''' + + +def test_loglevel_debug(caplog): + @swhmain.command(name='test') + @click.pass_context + def swhtest(ctx): + assert logging.root.level == 10 + click.echo('Hello SWH!') + + runner = CliRunner() + result = runner.invoke(swhmain, ['-l', 'DEBUG', 'test']) + assert result.exit_code == 0 + assert result.output.strip() == '''Hello SWH!''' + + +def test_aliased_command(): + @swhmain.command(name='canonical-test') + @click.pass_context + def swhtest(ctx): + 'A test command.' + click.echo('Hello SWH!') + swhmain.add_alias(swhtest, 'othername') + + runner = CliRunner() + + # check we have only 'canonical-test' listed in the usage help msg + result = runner.invoke(swhmain, ['-h']) + assert result.exit_code == 0 + assert 'canonical-test A test command.' in result.output + assert 'othername' not in result.output + + # check we can execute the cmd with 'canonical-test' + result = runner.invoke(swhmain, ['canonical-test']) + assert result.exit_code == 0 + assert result.output.strip() == '''Hello SWH!''' + + # check we can also execute the cmd with the alias 'othername' + result = runner.invoke(swhmain, ['othername']) + assert result.exit_code == 0 + assert result.output.strip() == '''Hello SWH!''' diff --git a/version.txt b/version.txt index 05e622d..b7a94df 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.60-0-g86eeb30 \ No newline at end of file +v0.0.61-0-gb1dff31 \ No newline at end of file