diff --git a/PKG-INFO b/PKG-INFO index c46f142..92db288 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,30 +1,30 @@ Metadata-Version: 2.1 Name: swh.core -Version: 0.0.60 +Version: 0.0.61 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN -Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Source, https://forge.softwareheritage.org/source/swh-core +Project-URL: Funding, https://www.softwareheritage.org/donate Description: swh-core ======== core library for swh's modules: - config parser - hash computations - serialization - logging mechanism Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Description-Content-Type: text/markdown Provides-Extra: db -Provides-Extra: testing Provides-Extra: http +Provides-Extra: testing diff --git a/requirements.txt b/requirements.txt index cd36a30..bd236c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,3 @@ PyYAML systemd-python -# these deps below are now handled in dedicated 'extras' and should be removed -# from this main requirement file ASAP -arrow -aiohttp -msgpack > 0.5 -psycopg2 -python-dateutil -requests -Flask -decorator diff --git a/swh.core.egg-info/PKG-INFO b/swh.core.egg-info/PKG-INFO index c46f142..92db288 100644 --- a/swh.core.egg-info/PKG-INFO +++ b/swh.core.egg-info/PKG-INFO @@ -1,30 +1,30 @@ Metadata-Version: 2.1 Name: swh.core -Version: 0.0.60 +Version: 0.0.61 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN -Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Source, https://forge.softwareheritage.org/source/swh-core +Project-URL: Funding, https://www.softwareheritage.org/donate Description: swh-core ======== core library for swh's modules: - config parser - hash computations - serialization - logging mechanism Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Description-Content-Type: text/markdown Provides-Extra: db -Provides-Extra: testing Provides-Extra: http +Provides-Extra: testing diff --git a/swh.core.egg-info/SOURCES.txt b/swh.core.egg-info/SOURCES.txt index 8590f71..32148b1 100644 --- a/swh.core.egg-info/SOURCES.txt +++ b/swh.core.egg-info/SOURCES.txt @@ -1,41 +1,44 @@ MANIFEST.in Makefile README.md requirements-swh.txt requirements.txt setup.py version.txt swh/__init__.py swh.core.egg-info/PKG-INFO swh.core.egg-info/SOURCES.txt swh.core.egg-info/dependency_links.txt swh.core.egg-info/entry_points.txt swh.core.egg-info/requires.txt swh.core.egg-info/top_level.txt swh/core/__init__.py swh/core/api_async.py swh/core/config.py swh/core/logger.py swh/core/statsd.py swh/core/tarball.py swh/core/utils.py swh/core/api/__init__.py swh/core/api/asynchronous.py swh/core/api/negotiation.py swh/core/api/serializers.py +swh/core/api/tests/__init__.py +swh/core/api/tests/server_testing.py +swh/core/api/tests/test_api.py +swh/core/api/tests/test_serializers.py swh/core/cli/__init__.py swh/core/cli/db.py swh/core/db/__init__.py swh/core/db/common.py swh/core/db/db_utils.py +swh/core/db/tests/__init__.py +swh/core/db/tests/conftest.py +swh/core/db/tests/db_testing.py +swh/core/db/tests/test_db.py swh/core/sql/log-schema.sql swh/core/tests/__init__.py -swh/core/tests/conftest.py -swh/core/tests/db_testing.py -swh/core/tests/server_testing.py -swh/core/tests/test_api.py +swh/core/tests/test_cli.py swh/core/tests/test_config.py -swh/core/tests/test_db.py -swh/core/tests/test_serializers.py swh/core/tests/test_statsd.py swh/core/tests/test_utils.py \ No newline at end of file diff --git a/swh.core.egg-info/requires.txt b/swh.core.egg-info/requires.txt index 1614931..131c64f 100644 --- a/swh.core.egg-info/requires.txt +++ b/swh.core.egg-info/requires.txt @@ -1,36 +1,29 @@ PyYAML systemd-python -arrow -aiohttp -msgpack>0.5 -psycopg2 -python-dateutil -requests -Flask -decorator [db] psycopg2 [http] aiohttp arrow decorator Flask msgpack>0.5 python-dateutil requests [testing] +Click pytest<4 pytest-postgresql requests-mock hypothesis>=3.11.0 psycopg2 aiohttp arrow decorator Flask msgpack>0.5 python-dateutil requests diff --git a/swh/core/api/__init__.py b/swh/core/api/__init__.py index 5c48358..0a23f66 100644 --- a/swh/core/api/__init__.py +++ b/swh/core/api/__init__.py @@ -1,322 +1,322 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import collections import functools import inspect import json import logging import pickle import requests import datetime from flask import Flask, Request, Response, request, abort from .serializers import (decode_response, encode_data_client as encode_data, msgpack_dumps, msgpack_loads, SWHJSONDecoder) from .negotiation import (Formatter as FormatterBase, Negotiator as NegotiatorBase, negotiate as _negotiate) logger = logging.getLogger(__name__) -# support for content negotation +# support for content negotiation class Negotiator(NegotiatorBase): def best_mimetype(self): return request.accept_mimetypes.best_match( self.accept_mimetypes, 'application/json') def _abort(self, status_code, err=None): return abort(status_code, err) def negotiate(formatter_cls, *args, **kwargs): return _negotiate(Negotiator, formatter_cls, *args, **kwargs) class Formatter(FormatterBase): def _make_response(self, body, content_type): return Response(body, content_type=content_type) class SWHJSONEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, (datetime.datetime, datetime.date)): return obj.isoformat() if isinstance(obj, datetime.timedelta): return str(obj) # Let the base class default method raise the TypeError return super().default(obj) class JSONFormatter(Formatter): format = 'json' mimetypes = ['application/json'] def render(self, obj): return json.dumps(obj, cls=SWHJSONEncoder) class MsgpackFormatter(Formatter): format = 'msgpack' mimetypes = ['application/x-msgpack'] def render(self, obj): return msgpack_dumps(obj) # base API classes class RemoteException(Exception): pass def remote_api_endpoint(path): def dec(f): f._endpoint_path = path return f return dec class APIError(Exception): """API Error""" def __str__(self): return ('An unexpected error occurred in the backend: {}' .format(self.args)) class MetaSWHRemoteAPI(type): """Metaclass for SWHRemoteAPI, which adds a method for each endpoint of the database it is designed to access. See for example :class:`swh.indexer.storage.api.client.RemoteStorage`""" def __new__(cls, name, bases, attributes): # For each method wrapped with @remote_api_endpoint in an API backend # (eg. :class:`swh.indexer.storage.IndexerStorage`), add a new # method in RemoteStorage, with the same documentation. # # Note that, despite the usage of decorator magic (eg. functools.wrap), # this never actually calls an IndexerStorage method. backend_class = attributes.get('backend_class', None) for base in bases: if backend_class is not None: break backend_class = getattr(base, 'backend_class', None) if backend_class: for (meth_name, meth) in backend_class.__dict__.items(): if hasattr(meth, '_endpoint_path'): cls.__add_endpoint(meth_name, meth, attributes) return super().__new__(cls, name, bases, attributes) @staticmethod def __add_endpoint(meth_name, meth, attributes): wrapped_meth = inspect.unwrap(meth) @functools.wraps(meth) # Copy signature and doc def meth_(*args, **kwargs): # Match arguments and parameters post_data = inspect.getcallargs( wrapped_meth, *args, **kwargs) # Remove arguments that should not be passed self = post_data.pop('self') post_data.pop('cur', None) post_data.pop('db', None) # Send the request. return self.post(meth._endpoint_path, post_data) attributes[meth_name] = meth_ class SWHRemoteAPI(metaclass=MetaSWHRemoteAPI): """Proxy to an internal SWH API """ backend_class = None """For each method of `backend_class` decorated with :func:`remote_api_endpoint`, a method with the same prototype and docstring will be added to this class. Calls to this new method will be translated into HTTP requests to a remote server. This backend class will never be instantiated, it only serves as a template.""" api_exception = APIError """The exception class to raise in case of communication error with the server.""" def __init__(self, url, api_exception=None, timeout=None, chunk_size=4096, **kwargs): if api_exception: self.api_exception = api_exception base_url = url if url.endswith('/') else url + '/' self.url = base_url self.session = requests.Session() self.timeout = timeout self.chunk_size = chunk_size def _url(self, endpoint): return '%s%s' % (self.url, endpoint) def raw_verb(self, verb, endpoint, **opts): if 'chunk_size' in opts: # if the chunk_size argument has been passed, consider the user # also wants stream=True, otherwise, what's the point. opts['stream'] = True if self.timeout and 'timeout' not in opts: opts['timeout'] = self.timeout try: return getattr(self.session, verb)( self._url(endpoint), **opts ) except requests.exceptions.ConnectionError as e: raise self.api_exception(e) def post(self, endpoint, data, **opts): if isinstance(data, (collections.Iterator, collections.Generator)): data = (encode_data(x) for x in data) else: data = encode_data(data) chunk_size = opts.pop('chunk_size', self.chunk_size) response = self.raw_verb( 'post', endpoint, data=data, headers={'content-type': 'application/x-msgpack', 'accept': 'application/x-msgpack'}, **opts) if opts.get('stream') or \ response.headers.get('transfer-encoding') == 'chunked': return response.iter_content(chunk_size) else: return self._decode_response(response) post_stream = post def get(self, endpoint, **opts): chunk_size = opts.pop('chunk_size', self.chunk_size) response = self.raw_verb( 'get', endpoint, headers={'accept': 'application/x-msgpack'}, **opts) if opts.get('stream') or \ response.headers.get('transfer-encoding') == 'chunked': return response.iter_content(chunk_size) else: return self._decode_response(response) def get_stream(self, endpoint, **opts): return self.get(endpoint, stream=True, **opts) def _decode_response(self, response): if response.status_code == 404: return None if response.status_code == 500: data = decode_response(response) if 'exception_pickled' in data: raise pickle.loads(data['exception_pickled']) else: raise RemoteException(data['exception']) # XXX: this breaks language-independence and should be # replaced by proper unserialization if response.status_code == 400: raise pickle.loads(decode_response(response)) elif response.status_code != 200: raise RemoteException( "Unexpected status code for API request: %s (%s)" % ( response.status_code, response.content, ) ) return decode_response(response) def __repr__(self): return '<{} url={}>'.format(self.__class__.__name__, self.url) class BytesRequest(Request): """Request with proper escaping of arbitrary byte sequences.""" encoding = 'utf-8' encoding_errors = 'surrogateescape' ENCODERS = { 'application/x-msgpack': msgpack_dumps, 'application/json': json.dumps, } def encode_data_server(data, content_type='application/x-msgpack'): encoded_data = ENCODERS[content_type](data) return Response( encoded_data, mimetype=content_type, ) def decode_request(request): content_type = request.mimetype data = request.get_data() if not data: return {} if content_type == 'application/x-msgpack': r = msgpack_loads(data) elif content_type == 'application/json': r = json.loads(data, cls=SWHJSONDecoder) else: raise ValueError('Wrong content type `%s` for API request' % content_type) return r def error_handler(exception, encoder): # XXX: this breaks language-independence and should be # replaced by proper serialization of errors logging.exception(exception) response = encoder(pickle.dumps(exception)) response.status_code = 400 return response class SWHServerAPIApp(Flask): """For each endpoint of the given `backend_class`, tells app.route to call a function that decodes the request and sends it to the backend object provided by the factory. :param Any backend_class: The class of the backend, which will be analyzed to look for API endpoints. :param Callable[[], backend_class] backend_factory: A function with no argument that returns an instance of `backend_class`.""" request_class = BytesRequest def __init__(self, *args, backend_class=None, backend_factory=None, **kwargs): super().__init__(*args, **kwargs) if backend_class is not None: if backend_factory is None: raise TypeError('Missing argument backend_factory') for (meth_name, meth) in backend_class.__dict__.items(): if hasattr(meth, '_endpoint_path'): self.__add_endpoint(meth_name, meth, backend_factory) def __add_endpoint(self, meth_name, meth, backend_factory): from flask import request @self.route('/'+meth._endpoint_path, methods=['POST']) @functools.wraps(meth) # Copy signature and doc def _f(): # Call the actual code obj_meth = getattr(backend_factory(), meth_name) return encode_data_server(obj_meth(**decode_request(request))) diff --git a/swh/core/api/tests/__init__.py b/swh/core/api/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/swh/core/tests/server_testing.py b/swh/core/api/tests/server_testing.py similarity index 100% rename from swh/core/tests/server_testing.py rename to swh/core/api/tests/server_testing.py diff --git a/swh/core/tests/test_api.py b/swh/core/api/tests/test_api.py similarity index 100% rename from swh/core/tests/test_api.py rename to swh/core/api/tests/test_api.py diff --git a/swh/core/tests/test_serializers.py b/swh/core/api/tests/test_serializers.py similarity index 100% rename from swh/core/tests/test_serializers.py rename to swh/core/api/tests/test_serializers.py diff --git a/swh/core/cli/__init__.py b/swh/core/cli/__init__.py index de82173..eefac67 100644 --- a/swh/core/cli/__init__.py +++ b/swh/core/cli/__init__.py @@ -1,40 +1,60 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import click import logging import pkg_resources +LOG_LEVEL_NAMES = ['NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] + +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) + logger = logging.getLogger(__name__) -CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) +class AliasedGroup(click.Group): + 'A simple Group that supports command aliases' + + @property + def aliases(self): + if not hasattr(self, '_aliases'): + self._aliases = {} + return self._aliases + + def get_command(self, ctx, cmd_name): + return super().get_command(ctx, self.aliases.get(cmd_name, cmd_name)) + + def add_alias(self, name, alias): + if not isinstance(name, str): + name = name.name + self.aliases[alias] = name -@click.group(context_settings=CONTEXT_SETTINGS) +@click.group(context_settings=CONTEXT_SETTINGS, cls=AliasedGroup) @click.option('--log-level', '-l', default='INFO', - type=click.Choice(logging._nameToLevel.keys()), + type=click.Choice(LOG_LEVEL_NAMES), help="Log level (default to INFO)") @click.pass_context def swh(ctx, log_level): - """Software Heritage Tool + """Command line interface for Software Heritage. """ - logger.setLevel(log_level) + log_level = logging.getLevelName(log_level) + logging.root.setLevel(log_level) ctx.ensure_object(dict) - ctx.obj['log_level'] = logging._nameToLevel[log_level] + ctx.obj['log_level'] = log_level def main(): logging.basicConfig() # load plugins that define cli sub commands for entry_point in pkg_resources.iter_entry_points('swh.cli.subcommands'): cmd = entry_point.load() swh.add_command(cmd, name=entry_point.name) return swh(auto_envvar_prefix='SWH') if __name__ == '__main__': main() diff --git a/swh/core/cli/db.py b/swh/core/cli/db.py index 27a1992..a258ea7 100755 --- a/swh/core/cli/db.py +++ b/swh/core/cli/db.py @@ -1,86 +1,87 @@ #!/usr/bin/env python3 # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import warnings warnings.filterwarnings("ignore") # noqa prevent psycopg from telling us sh*t import click +from swh.core.cli import CONTEXT_SETTINGS logger = logging.getLogger(__name__) -@click.command() +@click.command(context_settings=CONTEXT_SETTINGS) @click.argument('module', nargs=-1, required=True) @click.option('--db-name', '-d', help='Database name.', default='softwareheritage-dev', show_default=True) def db_init(module, db_name=None): """Initialise a database for the Software Heritage . By default, attempts to create the database first. Example: swh db-init -d swh-test storage If you want to specify non-default postgresql connection parameters, please provide them using standard environment variables. See psql(1) man page (section ENVIRONMENTS) for details. Example: PGPORT=5434 swh db-init indexer """ # put import statements here so we can keep startup time of the main swh # command as short as possible from os import path import glob from importlib import import_module from swh.core.utils import numfile_sortkey as sortkey from swh.core.tests.db_testing import ( pg_createdb, pg_restore, DB_DUMP_TYPES, swh_db_version ) logger.debug('db_init %s dn_name=%s', module, db_name) dump_files = [] for modname in module: if not modname.startswith('swh.'): modname = 'swh.{}'.format(modname) try: m = import_module(modname) except ImportError: raise click.BadParameter( 'Unable to load module {}'.format(modname)) sqldir = path.join(path.dirname(m.__file__), 'sql') if not path.isdir(sqldir): raise click.BadParameter( 'Module {} does not provide a db schema ' '(no sql/ dir)'.format(modname)) dump_files.extend(sorted(glob.glob(path.join(sqldir, '*.sql')), key=sortkey)) # Create the db (or fail silently if already existing) pg_createdb(db_name, check=False) # Try to retrieve the db version if any db_version = swh_db_version(db_name) if not db_version: # Initialize the db dump_files = [(x, DB_DUMP_TYPES[path.splitext(x)[1]]) for x in dump_files] for dump, dtype in dump_files: click.secho('Loading {}'.format(dump), fg='yellow') pg_restore(db_name, dump, dtype) db_version = swh_db_version(db_name) # TODO: Ideally migrate the version from db_version to the latest # db version click.secho('DONE database is {} version {}'.format(db_name, db_version), fg='green', bold=True) diff --git a/swh/core/db/__init__.py b/swh/core/db/__init__.py index da0e402..455f56d 100644 --- a/swh/core/db/__init__.py +++ b/swh/core/db/__init__.py @@ -1,193 +1,193 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import binascii import datetime import enum import json import os import threading from contextlib import contextmanager import psycopg2 import psycopg2.extras psycopg2.extras.register_uuid() def escape(data): if data is None: return '' if isinstance(data, bytes): return '\\x%s' % binascii.hexlify(data).decode('ascii') elif isinstance(data, str): return '"%s"' % data.replace('"', '""') elif isinstance(data, datetime.datetime): # We escape twice to make sure the string generated by # isoformat gets escaped return escape(data.isoformat()) elif isinstance(data, dict): return escape(json.dumps(data)) elif isinstance(data, list): return escape("{%s}" % ','.join(escape(d) for d in data)) elif isinstance(data, psycopg2.extras.Range): # We escape twice here too, so that we make sure # everything gets passed to copy properly return escape( '%s%s,%s%s' % ( '[' if data.lower_inc else '(', '-infinity' if data.lower_inf else escape(data.lower), 'infinity' if data.upper_inf else escape(data.upper), ']' if data.upper_inc else ')', ) ) elif isinstance(data, enum.IntEnum): return escape(int(data)) else: # We don't escape here to make sure we pass literals properly return str(data) def typecast_bytea(value, cur): if value is not None: data = psycopg2.BINARY(value, cur) return data.tobytes() class BaseDb: """Base class for swh.*.*Db. cf. swh.storage.db.Db, swh.archiver.db.ArchiverDb """ @classmethod def adapt_conn(cls, conn): """Makes psycopg2 use 'bytes' to decode bytea instead of 'memoryview', for this connection.""" cur = conn.cursor() cur.execute("SELECT null::bytea, null::bytea[]") bytea_oid = cur.description[0][1] bytea_array_oid = cur.description[1][1] t_bytes = psycopg2.extensions.new_type( (bytea_oid,), "bytea", typecast_bytea) psycopg2.extensions.register_type(t_bytes, conn) t_bytes_array = psycopg2.extensions.new_array_type( (bytea_array_oid,), "bytea[]", t_bytes) psycopg2.extensions.register_type(t_bytes_array, conn) @classmethod def connect(cls, *args, **kwargs): """factory method to create a DB proxy Accepts all arguments of psycopg2.connect; only some specific possibilities are reported below. Args: connstring: libpq2 connection string """ conn = psycopg2.connect(*args, **kwargs) return cls(conn) @classmethod def from_pool(cls, pool): conn = pool.getconn() return cls(conn, pool=pool) def __init__(self, conn, pool=None): """create a DB proxy Args: conn: psycopg2 connection to the SWH DB pool: psycopg2 pool of connections """ self.adapt_conn(conn) self.conn = conn self.pool = pool def put_conn(self): if self.pool: self.pool.putconn(self.conn) def cursor(self, cur_arg=None): """get a cursor: from cur_arg if given, or a fresh one otherwise meant to avoid boilerplate if/then/else in methods that proxy stored procedures """ if cur_arg is not None: return cur_arg else: return self.conn.cursor() _cursor = cursor # for bw compat @contextmanager def transaction(self): """context manager to execute within a DB transaction Yields: a psycopg2 cursor """ with self.conn.cursor() as cur: try: yield cur self.conn.commit() except Exception: if not self.conn.closed: self.conn.rollback() raise def copy_to(self, items, tblname, columns, cur=None, item_cb=None, default_values={}): """Copy items' entries to table tblname with columns information. Args: items (List[dict]): dictionaries of data to copy over tblname. tblname (str): destination table's name. columns ([str]): keys to access data in items and also the column names in the destination table. - default_values (dict): dictionnary of default values to use when + default_values (dict): dictionary of default values to use when inserting entried int the tblname table. cur: a db cursor; if not given, a new cursor will be created. item_cb (fn): optional function to apply to items's entry. """ read_file, write_file = os.pipe() def writer(): cursor = self.cursor(cur) with open(read_file, 'r') as f: cursor.copy_expert('COPY %s (%s) FROM STDIN CSV' % ( tblname, ', '.join(columns)), f) write_thread = threading.Thread(target=writer) write_thread.start() try: with open(write_file, 'w') as f: for d in items: if item_cb is not None: item_cb(d) line = [escape(d.get(k, default_values.get(k))) for k in columns] f.write(','.join(line)) f.write('\n') finally: # No problem bubbling up exceptions, but we still need to make sure # we finish copying, even though we're probably going to cancel the # transaction. write_thread.join() def mktemp(self, tblname, cur=None): self.cursor(cur).execute('SELECT swh_mktemp(%s)', (tblname,)) diff --git a/swh/core/db/tests/__init__.py b/swh/core/db/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/swh/core/tests/conftest.py b/swh/core/db/tests/conftest.py similarity index 100% rename from swh/core/tests/conftest.py rename to swh/core/db/tests/conftest.py diff --git a/swh/core/tests/db_testing.py b/swh/core/db/tests/db_testing.py similarity index 100% rename from swh/core/tests/db_testing.py rename to swh/core/db/tests/db_testing.py diff --git a/swh/core/tests/test_db.py b/swh/core/db/tests/test_db.py similarity index 98% rename from swh/core/tests/test_db.py rename to swh/core/db/tests/test_db.py index a9c70e8..355a384 100644 --- a/swh/core/tests/test_db.py +++ b/swh/core/db/tests/test_db.py @@ -1,102 +1,102 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os.path import tempfile import unittest from hypothesis import strategies, given import pytest from swh.core.db import BaseDb -from swh.core.tests.db_testing import ( +from .db_testing import ( SingleDbTestFixture, db_create, db_destroy, db_close, ) INIT_SQL = ''' create table test_table ( i int, txt text, bytes bytea ); ''' db_rows = strategies.lists(strategies.tuples( strategies.integers(-2147483648, +2147483647), strategies.text( alphabet=strategies.characters( blacklist_categories=['Cs'], # surrogates blacklist_characters=[ '\x00', # pgsql does not support the null codepoint '\r', # pgsql normalizes those ] ), ), strategies.binary(), )) @pytest.mark.db def test_connect(): db_name = db_create('test-db2', dumps=[]) try: db = BaseDb.connect('dbname=%s' % db_name) with db.cursor() as cur: cur.execute(INIT_SQL) cur.execute("insert into test_table values (1, %s, %s);", ('foo', b'bar')) cur.execute("select * from test_table;") assert list(cur) == [(1, 'foo', b'bar')] finally: db_close(db.conn) db_destroy(db_name) @pytest.mark.db class TestDb(SingleDbTestFixture, unittest.TestCase): TEST_DB_NAME = 'test-db' @classmethod def setUpClass(cls): with tempfile.TemporaryDirectory() as td: with open(os.path.join(td, 'init.sql'), 'a') as fd: fd.write(INIT_SQL) cls.TEST_DB_DUMP = os.path.join(td, '*.sql') super().setUpClass() def setUp(self): super().setUp() self.db = BaseDb(self.conn) def test_initialized(self): cur = self.db.cursor() cur.execute("insert into test_table values (1, %s, %s);", ('foo', b'bar')) cur.execute("select * from test_table;") self.assertEqual(list(cur), [(1, 'foo', b'bar')]) def test_reset_tables(self): cur = self.db.cursor() cur.execute("insert into test_table values (1, %s, %s);", ('foo', b'bar')) self.reset_db_tables('test-db') cur.execute("select * from test_table;") self.assertEqual(list(cur), []) @given(db_rows) def test_copy_to(self, data): # the table is not reset between runs by hypothesis self.reset_db_tables('test-db') items = [dict(zip(['i', 'txt', 'bytes'], item)) for item in data] self.db.copy_to(items, 'test_table', ['i', 'txt', 'bytes']) cur = self.db.cursor() cur.execute('select * from test_table;') self.assertCountEqual(list(cur), data) diff --git a/swh/core/tests/test_cli.py b/swh/core/tests/test_cli.py new file mode 100644 index 0000000..63f2538 --- /dev/null +++ b/swh/core/tests/test_cli.py @@ -0,0 +1,109 @@ +# + +import logging + +import click +from click.testing import CliRunner + +from swh.core.cli import swh as swhmain + + +help_msg = '''Usage: swh [OPTIONS] COMMAND [ARGS]... + + Command line interface for Software Heritage. + +Options: + -l, --log-level [NOTSET|DEBUG|INFO|WARNING|ERROR|CRITICAL] + Log level (default to INFO) + -h, --help Show this message and exit. +''' + + +def test_swh_help(): + runner = CliRunner() + result = runner.invoke(swhmain, ['-h']) + assert result.exit_code == 0 + assert result.output == help_msg + + result = runner.invoke(swhmain, ['--help']) + assert result.exit_code == 0 + assert result.output == help_msg + + +def test_command(): + @swhmain.command(name='test') + @click.pass_context + def swhtest(ctx): + click.echo('Hello SWH!') + + runner = CliRunner() + result = runner.invoke(swhmain, ['test']) + assert result.exit_code == 0 + assert result.output.strip() == 'Hello SWH!' + + +def test_loglevel_default(caplog): + @swhmain.command(name='test') + @click.pass_context + def swhtest(ctx): + assert logging.root.level == 20 + click.echo('Hello SWH!') + + runner = CliRunner() + result = runner.invoke(swhmain, ['test']) + assert result.exit_code == 0 + print(result.output) + assert result.output.strip() == '''Hello SWH!''' + + +def test_loglevel_error(caplog): + @swhmain.command(name='test') + @click.pass_context + def swhtest(ctx): + assert logging.root.level == 40 + click.echo('Hello SWH!') + + runner = CliRunner() + result = runner.invoke(swhmain, ['-l', 'ERROR', 'test']) + assert result.exit_code == 0 + assert result.output.strip() == '''Hello SWH!''' + + +def test_loglevel_debug(caplog): + @swhmain.command(name='test') + @click.pass_context + def swhtest(ctx): + assert logging.root.level == 10 + click.echo('Hello SWH!') + + runner = CliRunner() + result = runner.invoke(swhmain, ['-l', 'DEBUG', 'test']) + assert result.exit_code == 0 + assert result.output.strip() == '''Hello SWH!''' + + +def test_aliased_command(): + @swhmain.command(name='canonical-test') + @click.pass_context + def swhtest(ctx): + 'A test command.' + click.echo('Hello SWH!') + swhmain.add_alias(swhtest, 'othername') + + runner = CliRunner() + + # check we have only 'canonical-test' listed in the usage help msg + result = runner.invoke(swhmain, ['-h']) + assert result.exit_code == 0 + assert 'canonical-test A test command.' in result.output + assert 'othername' not in result.output + + # check we can execute the cmd with 'canonical-test' + result = runner.invoke(swhmain, ['canonical-test']) + assert result.exit_code == 0 + assert result.output.strip() == '''Hello SWH!''' + + # check we can also execute the cmd with the alias 'othername' + result = runner.invoke(swhmain, ['othername']) + assert result.exit_code == 0 + assert result.output.strip() == '''Hello SWH!''' diff --git a/version.txt b/version.txt index 05e622d..b7a94df 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.60-0-g86eeb30 \ No newline at end of file +v0.0.61-0-gb1dff31 \ No newline at end of file