diff --git a/requirements-db.txt b/requirements-db.txt new file mode 100644 --- /dev/null +++ b/requirements-db.txt @@ -0,0 +1,2 @@ +# requirements for swh.core.db +psycopg2 diff --git a/requirements.txt b/requirements-http.txt copy from requirements.txt copy to requirements-http.txt --- a/requirements.txt +++ b/requirements-http.txt @@ -1,11 +1,8 @@ -arrow +# requirements for swh.core.api aiohttp +arrow +decorator +Flask msgpack > 0.5 -psycopg2 python-dateutil -vcversioner -PyYAML requests -Flask -systemd-python -decorator diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,13 @@ +PyYAML +systemd-python + +# these deps below are now handled in dedicated 'extras' and should be removed +# from this main requirement file ASAP arrow aiohttp msgpack > 0.5 psycopg2 python-dateutil -vcversioner -PyYAML requests Flask -systemd-python decorator diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 --- a/setup.py +++ b/setup.py @@ -17,22 +17,23 @@ long_description = f.read() -def parse_requirements(name=None): - if name: - reqf = 'requirements-%s.txt' % name - else: - reqf = 'requirements.txt' - +def parse_requirements(*names): requirements = [] - if not os.path.exists(reqf): - return requirements + for name in names: + if name: + reqf = 'requirements-%s.txt' % name + else: + reqf = 'requirements.txt' + + if not os.path.exists(reqf): + return requirements - with open(reqf) as f: - for line in f.readlines(): - line = line.strip() - if not line or line.startswith('#'): - continue - requirements.append(line) + with open(reqf) as f: + for line in f.readlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + requirements.append(line) return requirements @@ -46,9 +47,13 @@ url='https://forge.softwareheritage.org/diffusion/DCORE/', packages=find_packages(), scripts=[], - install_requires=parse_requirements() + parse_requirements('swh'), + install_requires=parse_requirements(None, 'swh'), setup_requires=['vcversioner'], - extras_require={'testing': parse_requirements('test')}, + extras_require={ + 'testing': parse_requirements('test', 'db', 'http'), + 'db': parse_requirements('db'), + 'http': parse_requirements('http'), + }, vcversioner={}, include_package_data=True, entry_points=''' diff --git a/swh/core/logger.py b/swh/core/logger.py --- a/swh/core/logger.py +++ b/swh/core/logger.py @@ -5,11 +5,7 @@ import datetime import logging -import os -import socket -import psycopg2 -from psycopg2.extras import Json from systemd.journal import JournalHandler as _JournalHandler, send try: @@ -80,90 +76,6 @@ return str(value) -class PostgresHandler(logging.Handler): - """log handler that store messages in a Postgres DB - - See swh-core/swh/core/sql/log-schema.sql for the DB schema. - - All logging methods can be used as usual. Additionally, arbitrary metadata - can be passed to logging methods, requesting that they will be stored in - the DB as a single JSONB value. To do so, pass a dictionary to the 'extra' - kwarg of any logging method; all keys in that dictionary that start with - EXTRA_LOGDATA_PREFIX (currently: ``swh_``) will be extracted to form the - JSONB dictionary. The prefix will be stripped and not included in the DB. - - Note: the logger name will be used to fill the 'module' DB column. - - Sample usage:: - - logging.basicConfig(level=logging.INFO) - h = PostgresHandler('dbname=softwareheritage-log') - logging.getLogger().addHandler(h) - - logger.info('not so important notice', - extra={'swh_type': 'swh_logging_test', - 'swh_meditation': 'guru'}) - logger.warn('something weird just happened, did you see that?') - - """ - - def __init__(self, connstring): - """ - Create a Postgres log handler. - - Args: - config: configuration dictionary, with a key "log_db" containing a - libpq connection string to the log DB - """ - super().__init__() - - self.connstring = connstring - - self.fqdn = socket.getfqdn() # cache FQDN value - - def _connect(self): - return psycopg2.connect(self.connstring) - - def emit(self, record): - msg = self.format(record) - extra_data = get_extra_data(record) - - if 'task' in extra_data: - task_args = { - 'args': extra_data['task']['args'], - 'kwargs': extra_data['task']['kwargs'], - } - - try: - json_args = Json(task_args).getquoted() - except TypeError: - task_args = { - 'args': [''], - 'kwargs': {}, - } - else: - json_args_length = len(json_args) - if json_args_length >= 1000: - task_args = { - 'args': [''], - 'kwargs': {}, - } - - extra_data['task'].update(task_args) - - log_entry = (db_level_of_py_level(record.levelno), msg, - Json(extra_data), record.name, self.fqdn, - os.getpid()) - db = self._connect() - with db.cursor() as cur: - cur.execute('INSERT INTO log ' - '(level, message, data, src_module, src_host, src_pid)' - 'VALUES (%s, %s, %s, %s, %s, %s)', - log_entry) - db.commit() - db.close() - - class JournalHandler(_JournalHandler): def emit(self, record): """Write `record` as a journal event. diff --git a/swh/core/tests/test_logger.py b/swh/core/tests/test_logger.py deleted file mode 100644 --- a/swh/core/tests/test_logger.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (C) 2015-2018 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import logging -import os - -import pytest - -from swh.core.logger import PostgresHandler - -from swh.core.tests import SQL_DIR - -DUMP_FILE = os.path.join(SQL_DIR, 'log-schema.sql') - - -@pytest.fixture -def swh_db_logger(postgresql_proc, postgresql): - - cursor = postgresql.cursor() - with open(DUMP_FILE) as fobj: - cursor.execute(fobj.read()) - postgresql.commit() - modname = 'swh.core.tests.test_logger' - logger = logging.Logger(modname, logging.DEBUG) - dsn = 'postgresql://{user}@{host}:{port}/{dbname}'.format( - host=postgresql_proc.host, - port=postgresql_proc.port, - user='postgres', - dbname='tests') - logger.addHandler(PostgresHandler(dsn)) - return logger - - -@pytest.mark.db -def test_log(swh_db_logger, postgresql): - logger = swh_db_logger - modname = logger.name - - logger.info('notice', - extra={'swh_type': 'test entry', 'swh_data': 42}) - logger.warning('warning') - - with postgresql.cursor() as cur: - cur.execute('SELECT level, message, data, src_module FROM log') - db_log_entries = cur.fetchall() - - assert ('info', 'notice', {'type': 'test entry', 'data': 42}, - modname) in db_log_entries - assert ('warning', 'warning', {}, modname) in db_log_entries