diff --git a/MANIFEST.in b/MANIFEST.in index fde45c3..9fd29fa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,12 +1,7 @@ include Makefile -include requirements.txt -include requirements-db.txt -include requirements-http.txt -include requirements-logging.txt -include requirements-swh.txt -include requirements-test.txt +include requirements*.txt include version.txt recursive-include swh/core/sql *.sql recursive-include swh py.typed recursive-include swh/core/tests/data/ * recursive-include swh/core/tests/fixture/data/ * diff --git a/PKG-INFO b/PKG-INFO index 8fd28a3..a104bcf 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,89 +1,91 @@ Metadata-Version: 2.1 Name: swh.core -Version: 0.0.80 +Version: 0.0.81 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-core Description: swh-core ======== core library for swh's modules: - config parser - hash computations - serialization - logging mechanism - database connection - http-based RPC client/server Development ----------- We strongly recommend you to use a [virtualenv][1] if you want to run tests or hack the code. To set up your development environment: ``` (swh) user@host:~/swh-environment/swh-core$ pip install -e .[testing] ``` This will install every Python package needed to run this package's tests. Unit tests can be executed using [pytest][2] or [tox][3]. ``` (swh) user@host:~/swh-environment/swh-core$ pytest ============================== test session starts ============================== platform linux -- Python 3.7.3, pytest-3.10.1, py-1.8.0, pluggy-0.12.0 hypothesis profile 'default' -> database=DirectoryBasedExampleDatabase('/home/ddouard/src/swh-environment/swh-core/.hypothesis/examples') rootdir: /home/ddouard/src/swh-environment/swh-core, inifile: pytest.ini plugins: requests-mock-1.6.0, hypothesis-4.26.4, celery-4.3.0, postgresql-1.4.1 collected 89 items swh/core/api/tests/test_api.py .. [ 2%] swh/core/api/tests/test_async.py .... [ 6%] swh/core/api/tests/test_serializers.py ..... [ 12%] swh/core/db/tests/test_db.py .... [ 16%] swh/core/tests/test_cli.py ...... [ 23%] swh/core/tests/test_config.py .............. [ 39%] swh/core/tests/test_statsd.py ........................................... [ 87%] .... [ 92%] swh/core/tests/test_utils.py ....... [100%] ===================== 89 passed, 9 warnings in 6.94 seconds ===================== ``` Note: this git repository uses [pre-commit][4] hooks to ensure better and more consistent code. It should already be installed in your virtualenv (if not, just type `pip install pre-commit`). Make sure to activate it in your local copy of the git repository: ``` (swh) user@host:~/swh-environment/swh-core$ pre-commit install pre-commit installed at .git/hooks/pre-commit ``` Please read the [developer setup manual][5] for more information on how to hack on Software Heritage. [1]: https://virtualenv.pypa.io [2]: https://docs.pytest.org [3]: https://tox.readthedocs.io [4]: https://pre-commit.com [5]: https://docs.softwareheritage.org/devel/developer-setup.html Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Description-Content-Type: text/markdown -Provides-Extra: testing +Provides-Extra: testing-core Provides-Extra: logging Provides-Extra: db +Provides-Extra: testing-db Provides-Extra: http +Provides-Extra: testing diff --git a/requirements-http.txt b/requirements-http.txt index 67e8841..c66192b 100644 --- a/requirements-http.txt +++ b/requirements-http.txt @@ -1,9 +1,10 @@ # requirements for swh.core.api aiohttp aiohttp_utils >= 3.1.1 arrow decorator Flask msgpack > 0.5 python-dateutil requests +blinker # dependency of sentry-sdk[flask] diff --git a/requirements-test-db.txt b/requirements-test-db.txt new file mode 100644 index 0000000..cfd42eb --- /dev/null +++ b/requirements-test-db.txt @@ -0,0 +1 @@ +pytest-postgresql diff --git a/requirements-test.txt b/requirements-test.txt index 170e10c..6bf9fdc 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,5 @@ pytest -pytest-postgresql requests-mock hypothesis >= 3.11.0 pre-commit pytz diff --git a/requirements.txt b/requirements.txt index d08d15b..7c16399 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ Click Deprecated PyYAML +sentry-sdk diff --git a/setup.py b/setup.py index 43d5dca..3544b4d 100755 --- a/setup.py +++ b/setup.py @@ -1,83 +1,87 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os from setuptools import setup, find_packages from os import path from io import open here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() def parse_requirements(*names): requirements = [] for name in names: if name: reqf = 'requirements-%s.txt' % name else: reqf = 'requirements.txt' if not os.path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith('#'): continue requirements.append(line) return requirements setup( name='swh.core', description='Software Heritage core utilities', long_description=long_description, long_description_content_type='text/markdown', author='Software Heritage developers', author_email='swh-devel@inria.fr', url='https://forge.softwareheritage.org/diffusion/DCORE/', packages=find_packages(), py_modules=['pytest_swh_core'], scripts=[], install_requires=parse_requirements(None, 'swh'), setup_requires=['vcversioner'], extras_require={ - 'testing': parse_requirements('test', 'db', 'http', 'logging'), + 'testing-core': parse_requirements('test'), 'logging': parse_requirements('logging'), 'db': parse_requirements('db'), + 'testing-db': parse_requirements('test-db'), 'http': parse_requirements('http'), + # kitchen sink, please do not use + 'testing': parse_requirements('test', 'test-db', 'db', 'http', + 'logging'), }, vcversioner={}, include_package_data=True, entry_points=''' [console_scripts] swh=swh.core.cli:main swh-db-init=swh.core.cli.db:db_init [swh.cli.subcommands] db=swh.core.cli.db:db db-init=swh.core.cli.db:db_init [pytest11] pytest_swh_core = swh.core.pytest_plugin ''', classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ 'Bug Reports': 'https://forge.softwareheritage.org/maniphest', 'Funding': 'https://www.softwareheritage.org/donate', 'Source': 'https://forge.softwareheritage.org/source/swh-core', }, ) diff --git a/swh.core.egg-info/PKG-INFO b/swh.core.egg-info/PKG-INFO index 8fd28a3..a104bcf 100644 --- a/swh.core.egg-info/PKG-INFO +++ b/swh.core.egg-info/PKG-INFO @@ -1,89 +1,91 @@ Metadata-Version: 2.1 Name: swh.core -Version: 0.0.80 +Version: 0.0.81 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-core Description: swh-core ======== core library for swh's modules: - config parser - hash computations - serialization - logging mechanism - database connection - http-based RPC client/server Development ----------- We strongly recommend you to use a [virtualenv][1] if you want to run tests or hack the code. To set up your development environment: ``` (swh) user@host:~/swh-environment/swh-core$ pip install -e .[testing] ``` This will install every Python package needed to run this package's tests. Unit tests can be executed using [pytest][2] or [tox][3]. ``` (swh) user@host:~/swh-environment/swh-core$ pytest ============================== test session starts ============================== platform linux -- Python 3.7.3, pytest-3.10.1, py-1.8.0, pluggy-0.12.0 hypothesis profile 'default' -> database=DirectoryBasedExampleDatabase('/home/ddouard/src/swh-environment/swh-core/.hypothesis/examples') rootdir: /home/ddouard/src/swh-environment/swh-core, inifile: pytest.ini plugins: requests-mock-1.6.0, hypothesis-4.26.4, celery-4.3.0, postgresql-1.4.1 collected 89 items swh/core/api/tests/test_api.py .. [ 2%] swh/core/api/tests/test_async.py .... [ 6%] swh/core/api/tests/test_serializers.py ..... [ 12%] swh/core/db/tests/test_db.py .... [ 16%] swh/core/tests/test_cli.py ...... [ 23%] swh/core/tests/test_config.py .............. [ 39%] swh/core/tests/test_statsd.py ........................................... [ 87%] .... [ 92%] swh/core/tests/test_utils.py ....... [100%] ===================== 89 passed, 9 warnings in 6.94 seconds ===================== ``` Note: this git repository uses [pre-commit][4] hooks to ensure better and more consistent code. It should already be installed in your virtualenv (if not, just type `pip install pre-commit`). Make sure to activate it in your local copy of the git repository: ``` (swh) user@host:~/swh-environment/swh-core$ pre-commit install pre-commit installed at .git/hooks/pre-commit ``` Please read the [developer setup manual][5] for more information on how to hack on Software Heritage. [1]: https://virtualenv.pypa.io [2]: https://docs.pytest.org [3]: https://tox.readthedocs.io [4]: https://pre-commit.com [5]: https://docs.softwareheritage.org/devel/developer-setup.html Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Description-Content-Type: text/markdown -Provides-Extra: testing +Provides-Extra: testing-core Provides-Extra: logging Provides-Extra: db +Provides-Extra: testing-db Provides-Extra: http +Provides-Extra: testing diff --git a/swh.core.egg-info/SOURCES.txt b/swh.core.egg-info/SOURCES.txt index 70ed005..2100d5f 100644 --- a/swh.core.egg-info/SOURCES.txt +++ b/swh.core.egg-info/SOURCES.txt @@ -1,68 +1,71 @@ MANIFEST.in Makefile README.md requirements-db.txt requirements-http.txt requirements-logging.txt requirements-swh.txt +requirements-test-db.txt requirements-test.txt requirements.txt setup.py version.txt swh/__init__.py swh.core.egg-info/PKG-INFO swh.core.egg-info/SOURCES.txt swh.core.egg-info/dependency_links.txt swh.core.egg-info/entry_points.txt swh.core.egg-info/requires.txt swh.core.egg-info/top_level.txt swh/core/__init__.py swh/core/api_async.py swh/core/config.py swh/core/logger.py swh/core/py.typed swh/core/pytest_plugin.py swh/core/statsd.py swh/core/tarball.py swh/core/utils.py swh/core/api/__init__.py swh/core/api/asynchronous.py +swh/core/api/gunicorn_config.py swh/core/api/negotiation.py swh/core/api/serializers.py swh/core/api/tests/__init__.py swh/core/api/tests/server_testing.py swh/core/api/tests/test_async.py +swh/core/api/tests/test_gunicorn.py swh/core/api/tests/test_rpc_client.py swh/core/api/tests/test_rpc_client_server.py swh/core/api/tests/test_rpc_server.py swh/core/api/tests/test_serializers.py swh/core/cli/__init__.py swh/core/cli/db.py swh/core/db/__init__.py swh/core/db/common.py swh/core/db/db_utils.py swh/core/db/tests/__init__.py swh/core/db/tests/conftest.py swh/core/db/tests/db_testing.py swh/core/db/tests/test_cli.py swh/core/db/tests/test_db.py swh/core/sql/log-schema.sql swh/core/tests/__init__.py swh/core/tests/test_cli.py swh/core/tests/test_config.py swh/core/tests/test_logger.py swh/core/tests/test_pytest_plugin.py swh/core/tests/test_statsd.py swh/core/tests/test_tarball.py swh/core/tests/test_utils.py swh/core/tests/data/http_example.com/something.json swh/core/tests/data/https_example.com/file.json swh/core/tests/data/https_example.com/file.json,name=doe,firstname=jane swh/core/tests/data/https_example.com/file.json_visit1 swh/core/tests/data/https_example.com/other.json swh/core/tests/data/https_forge.s.o/api_diffusion,attachments[uris]=1 swh/core/tests/data/https_www.reference.com/web,q=What+Is+an+Example+of+a+URL?,qo=contentPageRelatedSearch,o=600605,l=dir,sga=1 swh/core/tests/fixture/__init__.py swh/core/tests/fixture/conftest.py swh/core/tests/fixture/test_pytest_plugin.py swh/core/tests/fixture/data/https_example.com/file.json \ No newline at end of file diff --git a/swh.core.egg-info/requires.txt b/swh.core.egg-info/requires.txt index 3e64a99..0ab6f15 100644 --- a/swh.core.egg-info/requires.txt +++ b/swh.core.egg-info/requires.txt @@ -1,37 +1,50 @@ Click Deprecated PyYAML +sentry-sdk [db] psycopg2 [http] aiohttp aiohttp_utils>=3.1.1 arrow decorator Flask msgpack>0.5 python-dateutil requests +blinker [logging] systemd-python [testing] pytest -pytest-postgresql requests-mock hypothesis>=3.11.0 pre-commit pytz +pytest-postgresql psycopg2 aiohttp aiohttp_utils>=3.1.1 arrow decorator Flask msgpack>0.5 python-dateutil requests +blinker systemd-python + +[testing-core] +pytest +requests-mock +hypothesis>=3.11.0 +pre-commit +pytz + +[testing-db] +pytest-postgresql diff --git a/swh/core/api/gunicorn_config.py b/swh/core/api/gunicorn_config.py new file mode 100644 index 0000000..ea1c6db --- /dev/null +++ b/swh/core/api/gunicorn_config.py @@ -0,0 +1,44 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +"""Default values for gunicorn's configuration. + +Other packages may override them by importing `*` from this module +and redefining functions and variables they want. + +May be imported by gunicorn using +`--config 'python:swh.core.api.gunicorn_config'`.""" + +import os + + +def _init_sentry( + sentry_dsn, *, flask=True, integrations=None, extra_kwargs={}): + import sentry_sdk + + integrations = integrations or [] + + if flask: + from sentry_sdk.integrations.flask import FlaskIntegration + integrations.append(FlaskIntegration()) + + sentry_sdk.init( + dsn=sentry_dsn, + integrations=integrations, + debug=bool(os.environ.get('SWH_SENTRY_DEBUG')), + **extra_kwargs, + ) + + +def post_fork( + server, worker, *, default_sentry_dsn=None, flask=True, + sentry_integrations=None, extra_sentry_kwargs={}): + + # Initializes sentry as soon as possible in gunicorn's worker processes. + sentry_dsn = os.environ.get('SWH_SENTRY_DSN', default_sentry_dsn) + if sentry_dsn: + _init_sentry( + sentry_dsn, flask=flask, integrations=sentry_integrations, + extra_kwargs=extra_sentry_kwargs) diff --git a/swh/core/api/tests/test_gunicorn.py b/swh/core/api/tests/test_gunicorn.py new file mode 100644 index 0000000..d652cb7 --- /dev/null +++ b/swh/core/api/tests/test_gunicorn.py @@ -0,0 +1,77 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import os +from unittest.mock import patch + +import swh.core.api.gunicorn_config as gunicorn_config + + +def test_post_fork_default(): + with patch('sentry_sdk.init') as sentry_sdk_init: + gunicorn_config.post_fork(None, None) + + sentry_sdk_init.assert_not_called() + + +def test_post_fork_with_dsn_env(): + flask_integration = object() # unique object to check for equality + with patch('sentry_sdk.integrations.flask.FlaskIntegration', + new=lambda: flask_integration): + with patch('sentry_sdk.init') as sentry_sdk_init: + with patch.dict(os.environ, {'SWH_SENTRY_DSN': 'test_dsn'}): + gunicorn_config.post_fork(None, None) + + sentry_sdk_init.assert_called_once_with( + dsn='test_dsn', + integrations=[flask_integration], + debug=False, + ) + + +def test_post_fork_debug(): + flask_integration = object() # unique object to check for equality + with patch('sentry_sdk.integrations.flask.FlaskIntegration', + new=lambda: flask_integration): + with patch('sentry_sdk.init') as sentry_sdk_init: + with patch.dict(os.environ, {'SWH_SENTRY_DSN': 'test_dsn', + 'SWH_SENTRY_DEBUG': '1'}): + gunicorn_config.post_fork(None, None) + + sentry_sdk_init.assert_called_once_with( + dsn='test_dsn', + integrations=[flask_integration], + debug=True, + ) + + +def test_post_fork_no_flask(): + with patch('sentry_sdk.init') as sentry_sdk_init: + with patch.dict(os.environ, {'SWH_SENTRY_DSN': 'test_dsn'}): + gunicorn_config.post_fork(None, None, flask=False) + + sentry_sdk_init.assert_called_once_with( + dsn='test_dsn', + integrations=[], + debug=False, + ) + + +def test_post_fork_extras(): + flask_integration = object() # unique object to check for equality + with patch('sentry_sdk.integrations.flask.FlaskIntegration', + new=lambda: flask_integration): + with patch('sentry_sdk.init') as sentry_sdk_init: + with patch.dict(os.environ, {'SWH_SENTRY_DSN': 'test_dsn'}): + gunicorn_config.post_fork( + None, None, sentry_integrations=['foo'], + extra_sentry_kwargs={'bar': 'baz'}) + + sentry_sdk_init.assert_called_once_with( + dsn='test_dsn', + integrations=['foo', flask_integration], + debug=False, + bar='baz', + ) diff --git a/swh/core/cli/__init__.py b/swh/core/cli/__init__.py index 685f2e2..bb10f2b 100644 --- a/swh/core/cli/__init__.py +++ b/swh/core/cli/__init__.py @@ -1,111 +1,115 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import logging.config import signal import click import pkg_resources +import sentry_sdk import yaml LOG_LEVEL_NAMES = ['NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) logger = logging.getLogger(__name__) class AliasedGroup(click.Group): '''A simple Group that supports command aliases, as well as notes related to options''' def __init__(self, name=None, commands=None, **attrs): self.option_notes = attrs.pop('option_notes', None) + self.aliases = {} super().__init__(name, commands, **attrs) - @property - def aliases(self): - if not hasattr(self, '_aliases'): - self._aliases = {} - return self._aliases - def get_command(self, ctx, cmd_name): return super().get_command(ctx, self.aliases.get(cmd_name, cmd_name)) def add_alias(self, name, alias): if not isinstance(name, str): name = name.name self.aliases[alias] = name def format_options(self, ctx, formatter): click.Command.format_options(self, ctx, formatter) if self.option_notes: with formatter.section('Notes'): formatter.write_text(self.option_notes) self.format_commands(ctx, formatter) def clean_exit_on_signal(signal, frame): """Raise a SystemExit exception to let command-line clients wind themselves down on exit""" raise SystemExit(0) @click.group( context_settings=CONTEXT_SETTINGS, cls=AliasedGroup, option_notes='''\ If both options are present, --log-level will override the root logger configuration set in --log-config. The --log-config YAML must conform to the logging.config.dictConfig schema documented at https://docs.python.org/3/library/logging.config.html. ''' ) @click.option('--log-level', '-l', default=None, type=click.Choice(LOG_LEVEL_NAMES), help="Log level (defaults to INFO).") @click.option('--log-config', default=None, type=click.File('r'), help="Python yaml logging configuration file.") +@click.option('--sentry-dsn', default=None, + help="DSN of the Sentry instance to report to") +@click.option('--sentry-debug/--no-sentry-debug', + default=False, hidden=True, + help="Enable debugging of sentry") @click.pass_context -def swh(ctx, log_level, log_config): +def swh(ctx, log_level, log_config, sentry_dsn, sentry_debug): """Command line interface for Software Heritage. """ signal.signal(signal.SIGTERM, clean_exit_on_signal) signal.signal(signal.SIGINT, clean_exit_on_signal) + if sentry_dsn: + sentry_sdk.init(dsn=sentry_dsn, debug=sentry_debug) + if log_level is None and log_config is None: log_level = 'INFO' if log_config: logging.config.dictConfig(yaml.safe_load(log_config.read())) if log_level: log_level = logging.getLevelName(log_level) logging.root.setLevel(log_level) ctx.ensure_object(dict) ctx.obj['log_level'] = log_level def main(): # Even though swh() sets up logging, we need an earlier basic logging setup # for the next few logging statements logging.basicConfig() # load plugins that define cli sub commands for entry_point in pkg_resources.iter_entry_points('swh.cli.subcommands'): try: cmd = entry_point.load() swh.add_command(cmd, name=entry_point.name) except Exception as e: logger.warning('Could not load subcommand %s: %s', entry_point.name, str(e)) return swh(auto_envvar_prefix='SWH') if __name__ == '__main__': main() diff --git a/swh/core/db/tests/test_cli.py b/swh/core/db/tests/test_cli.py index ee0d307..118ff7d 100644 --- a/swh/core/db/tests/test_cli.py +++ b/swh/core/db/tests/test_cli.py @@ -1,57 +1,57 @@ # from click.testing import CliRunner -from swh.core.cli import swh as swhmain from swh.core.cli.db import db as swhdb help_msg = '''Usage: swh [OPTIONS] COMMAND [ARGS]... Command line interface for Software Heritage. Options: -l, --log-level [NOTSET|DEBUG|INFO|WARNING|ERROR|CRITICAL] Log level (defaults to INFO). --log-config FILENAME Python yaml logging configuration file. + --sentry-dsn TEXT DSN of the Sentry instance to report to -h, --help Show this message and exit. Notes: If both options are present, --log-level will override the root logger configuration set in --log-config. The --log-config YAML must conform to the logging.config.dictConfig schema documented at https://docs.python.org/3/library/logging.config.html. Commands: db Software Heritage database generic tools. ''' -def test_swh_help(): +def test_swh_help(swhmain): swhmain.add_command(swhdb) runner = CliRunner() result = runner.invoke(swhmain, ['-h']) assert result.exit_code == 0 assert result.output == help_msg help_db_msg = '''Usage: swh db [OPTIONS] COMMAND [ARGS]... Software Heritage database generic tools. Options: -C, --config-file FILE Configuration file. -h, --help Show this message and exit. Commands: init Initialize the database for every Software Heritage module found in... ''' -def test_swh_db_help(): +def test_swh_db_help(swhmain): swhmain.add_command(swhdb) runner = CliRunner() result = runner.invoke(swhmain, ['db', '-h']) assert result.exit_code == 0 assert result.output == help_db_msg diff --git a/swh/core/tests/test_cli.py b/swh/core/tests/test_cli.py index f71462b..7faed10 100644 --- a/swh/core/tests/test_cli.py +++ b/swh/core/tests/test_cli.py @@ -1,197 +1,259 @@ -# +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information import logging import textwrap +from unittest.mock import patch import click from click.testing import CliRunner import pytest -from swh.core.cli import swh as swhmain - help_msg = '''Usage: swh [OPTIONS] COMMAND [ARGS]... Command line interface for Software Heritage. Options: -l, --log-level [NOTSET|DEBUG|INFO|WARNING|ERROR|CRITICAL] Log level (defaults to INFO). --log-config FILENAME Python yaml logging configuration file. + --sentry-dsn TEXT DSN of the Sentry instance to report to -h, --help Show this message and exit. Notes: If both options are present, --log-level will override the root logger configuration set in --log-config. The --log-config YAML must conform to the logging.config.dictConfig schema documented at https://docs.python.org/3/library/logging.config.html. ''' -def test_swh_help(): +def test_swh_help(swhmain): runner = CliRunner() result = runner.invoke(swhmain, ['-h']) assert result.exit_code == 0 assert result.output.startswith(help_msg) result = runner.invoke(swhmain, ['--help']) assert result.exit_code == 0 assert result.output.startswith(help_msg) -def test_command(): +def test_command(swhmain): @swhmain.command(name='test') @click.pass_context def swhtest(ctx): click.echo('Hello SWH!') runner = CliRunner() - result = runner.invoke(swhmain, ['test']) + with patch('sentry_sdk.init') as sentry_sdk_init: + result = runner.invoke(swhmain, ['test']) + sentry_sdk_init.assert_not_called() assert result.exit_code == 0 assert result.output.strip() == 'Hello SWH!' -def test_loglevel_default(caplog): +def test_loglevel_default(caplog, swhmain): @swhmain.command(name='test') @click.pass_context def swhtest(ctx): assert logging.root.level == 20 click.echo('Hello SWH!') runner = CliRunner() result = runner.invoke(swhmain, ['test']) assert result.exit_code == 0 print(result.output) assert result.output.strip() == '''Hello SWH!''' -def test_loglevel_error(caplog): +def test_loglevel_error(caplog, swhmain): @swhmain.command(name='test') @click.pass_context def swhtest(ctx): assert logging.root.level == 40 click.echo('Hello SWH!') runner = CliRunner() result = runner.invoke(swhmain, ['-l', 'ERROR', 'test']) assert result.exit_code == 0 assert result.output.strip() == '''Hello SWH!''' -def test_loglevel_debug(caplog): +def test_loglevel_debug(caplog, swhmain): @swhmain.command(name='test') @click.pass_context def swhtest(ctx): assert logging.root.level == 10 click.echo('Hello SWH!') runner = CliRunner() result = runner.invoke(swhmain, ['-l', 'DEBUG', 'test']) assert result.exit_code == 0 assert result.output.strip() == '''Hello SWH!''' +def test_sentry(swhmain): + @swhmain.command(name='test') + @click.pass_context + def swhtest(ctx): + click.echo('Hello SWH!') + + runner = CliRunner() + with patch('sentry_sdk.init') as sentry_sdk_init: + result = runner.invoke(swhmain, ['--sentry-dsn', 'test_dsn', 'test']) + assert result.exit_code == 0 + assert result.output.strip() == '''Hello SWH!''' + sentry_sdk_init.assert_called_once_with( + dsn='test_dsn', + debug=False, + ) + + +def test_sentry_debug(swhmain): + @swhmain.command(name='test') + @click.pass_context + def swhtest(ctx): + click.echo('Hello SWH!') + + runner = CliRunner() + with patch('sentry_sdk.init') as sentry_sdk_init: + result = runner.invoke( + swhmain, ['--sentry-dsn', 'test_dsn', '--sentry-debug', 'test']) + assert result.exit_code == 0 + assert result.output.strip() == '''Hello SWH!''' + sentry_sdk_init.assert_called_once_with( + dsn='test_dsn', + debug=True, + ) + + +def test_sentry_env(swhmain): + @swhmain.command(name='test') + @click.pass_context + def swhtest(ctx): + click.echo('Hello SWH!') + + runner = CliRunner() + with patch('sentry_sdk.init') as sentry_sdk_init: + env = { + 'SWH_SENTRY_DSN': 'test_dsn', + 'SWH_SENTRY_DEBUG': '1', + } + result = runner.invoke( + swhmain, ['test'], env=env, auto_envvar_prefix='SWH') + assert result.exit_code == 0 + assert result.output.strip() == '''Hello SWH!''' + sentry_sdk_init.assert_called_once_with( + dsn='test_dsn', + debug=True, + ) + + @pytest.fixture def log_config_path(tmp_path): log_config = textwrap.dedent('''\ --- version: 1 formatters: formatter: format: 'custom format:%(name)s:%(levelname)s:%(message)s' handlers: console: class: logging.StreamHandler stream: ext://sys.stdout formatter: formatter level: DEBUG root: level: DEBUG handlers: - console loggers: dontshowdebug: level: INFO ''') (tmp_path / 'log_config.yml').write_text(log_config) yield str(tmp_path / 'log_config.yml') -def test_log_config(caplog, log_config_path): +def test_log_config(caplog, log_config_path, swhmain): @swhmain.command(name='test') @click.pass_context def swhtest(ctx): logging.debug('Root log debug') logging.info('Root log info') logging.getLogger('dontshowdebug').debug('Not shown') logging.getLogger('dontshowdebug').info('Shown') runner = CliRunner() result = runner.invoke( swhmain, [ '--log-config', log_config_path, 'test', ], ) assert result.exit_code == 0 assert result.output.strip() == '\n'.join([ 'custom format:root:DEBUG:Root log debug', 'custom format:root:INFO:Root log info', 'custom format:dontshowdebug:INFO:Shown', ]) -def test_log_config_log_level_interaction(caplog, log_config_path): +def test_log_config_log_level_interaction(caplog, log_config_path, swhmain): @swhmain.command(name='test') @click.pass_context def swhtest(ctx): logging.debug('Root log debug') logging.info('Root log info') logging.getLogger('dontshowdebug').debug('Not shown') logging.getLogger('dontshowdebug').info('Shown') runner = CliRunner() result = runner.invoke( swhmain, [ '--log-config', log_config_path, '--log-level', 'INFO', 'test', ], ) assert result.exit_code == 0 assert result.output.strip() == '\n'.join([ 'custom format:root:INFO:Root log info', 'custom format:dontshowdebug:INFO:Shown', ]) -def test_aliased_command(): +def test_aliased_command(swhmain): @swhmain.command(name='canonical-test') @click.pass_context def swhtest(ctx): 'A test command.' click.echo('Hello SWH!') swhmain.add_alias(swhtest, 'othername') runner = CliRunner() # check we have only 'canonical-test' listed in the usage help msg result = runner.invoke(swhmain, ['-h']) assert result.exit_code == 0 assert 'canonical-test A test command.' in result.output assert 'othername' not in result.output # check we can execute the cmd with 'canonical-test' result = runner.invoke(swhmain, ['canonical-test']) assert result.exit_code == 0 assert result.output.strip() == '''Hello SWH!''' # check we can also execute the cmd with the alias 'othername' result = runner.invoke(swhmain, ['othername']) assert result.exit_code == 0 assert result.output.strip() == '''Hello SWH!''' diff --git a/version.txt b/version.txt index 5f98644..775beca 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.80-0-g155c0b2 \ No newline at end of file +v0.0.81-0-g139e3d3 \ No newline at end of file