diff --git a/requirements-db.txt b/requirements-db.txt new file mode 100644 --- /dev/null +++ b/requirements-db.txt @@ -0,0 +1,2 @@ +# requirements for swh.core.db +psycopg2 diff --git a/requirements.txt b/requirements-http.txt copy from requirements.txt copy to requirements-http.txt --- a/requirements.txt +++ b/requirements-http.txt @@ -1,11 +1,8 @@ -arrow +# requirements for swh.core.api aiohttp +arrow +decorator +Flask msgpack > 0.5 -psycopg2 python-dateutil -vcversioner -PyYAML requests -Flask -systemd-python -decorator diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,13 @@ +PyYAML +systemd-python + +# these deps below are now handled in dedicated 'extras' and should be removed +# from this main requirement file ASAP arrow aiohttp msgpack > 0.5 psycopg2 python-dateutil -vcversioner -PyYAML requests Flask -systemd-python decorator diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 --- a/setup.py +++ b/setup.py @@ -17,22 +17,23 @@ long_description = f.read() -def parse_requirements(name=None): - if name: - reqf = 'requirements-%s.txt' % name - else: - reqf = 'requirements.txt' - +def parse_requirements(*names): requirements = [] - if not os.path.exists(reqf): - return requirements + for name in names: + if name: + reqf = 'requirements-%s.txt' % name + else: + reqf = 'requirements.txt' + + if not os.path.exists(reqf): + return requirements - with open(reqf) as f: - for line in f.readlines(): - line = line.strip() - if not line or line.startswith('#'): - continue - requirements.append(line) + with open(reqf) as f: + for line in f.readlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + requirements.append(line) return requirements @@ -46,9 +47,13 @@ url='https://forge.softwareheritage.org/diffusion/DCORE/', packages=find_packages(), scripts=[], - install_requires=parse_requirements() + parse_requirements('swh'), + install_requires=parse_requirements(None, 'swh'), setup_requires=['vcversioner'], - extras_require={'testing': parse_requirements('test')}, + extras_require={ + 'testing': parse_requirements('test', 'db', 'http'), + 'db': parse_requirements('db'), + 'http': parse_requirements('http'), + }, vcversioner={}, include_package_data=True, entry_points=''' diff --git a/swh/core/logger.py b/swh/core/db/logger.py copy from swh/core/logger.py copy to swh/core/db/logger.py --- a/swh/core/logger.py +++ b/swh/core/db/logger.py @@ -3,81 +3,14 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import datetime import logging import os import socket import psycopg2 from psycopg2.extras import Json -from systemd.journal import JournalHandler as _JournalHandler, send -try: - from celery import current_task -except ImportError: - current_task = None - - -EXTRA_LOGDATA_PREFIX = 'swh_' - - -def db_level_of_py_level(lvl): - """convert a log level of the logging module to a log level suitable for the - logging Postgres DB - - """ - return logging.getLevelName(lvl).lower() - - -def get_extra_data(record, task_args=True): - """Get the extra data to insert to the database from the logging record""" - log_data = record.__dict__ - - extra_data = {k[len(EXTRA_LOGDATA_PREFIX):]: v - for k, v in log_data.items() - if k.startswith(EXTRA_LOGDATA_PREFIX)} - - args = log_data.get('args') - if args: - extra_data['logging_args'] = args - - # Retrieve Celery task info - if current_task and current_task.request: - extra_data['task'] = { - 'id': current_task.request.id, - 'name': current_task.name, - } - if task_args: - extra_data['task'].update({ - 'kwargs': current_task.request.kwargs, - 'args': current_task.request.args, - }) - - return extra_data - - -def flatten(data, separator='_'): - """Flatten the data dictionary into a flat structure""" - def inner_flatten(data, prefix): - if isinstance(data, dict): - for key, value in data.items(): - yield from inner_flatten(value, prefix + [key]) - elif isinstance(data, (list, tuple)): - for key, value in enumerate(data): - yield from inner_flatten(value, prefix + [str(key)]) - else: - yield prefix, data - - for path, value in inner_flatten(data, []): - yield separator.join(path), value - - -def stringify(value): - """Convert value to string""" - if isinstance(value, datetime.datetime): - return value.isoformat() - - return str(value) +from swh.core.logger import db_level_of_py_level, get_extra_data class PostgresHandler(logging.Handler): @@ -162,31 +95,3 @@ log_entry) db.commit() db.close() - - -class JournalHandler(_JournalHandler): - def emit(self, record): - """Write `record` as a journal event. - - MESSAGE is taken from the message provided by the user, and PRIORITY, - LOGGER, THREAD_NAME, CODE_{FILE,LINE,FUNC} fields are appended - automatically. In addition, record.MESSAGE_ID will be used if present. - """ - try: - extra_data = flatten(get_extra_data(record, task_args=False)) - extra_data = { - (EXTRA_LOGDATA_PREFIX + key).upper(): stringify(value) - for key, value in extra_data - } - msg = self.format(record) - pri = self.mapPriority(record.levelno) - send(msg, - PRIORITY=format(pri), - LOGGER=record.name, - THREAD_NAME=record.threadName, - CODE_FILE=record.pathname, - CODE_LINE=record.lineno, - CODE_FUNC=record.funcName, - **extra_data) - except Exception: - self.handleError(record) diff --git a/swh/core/logger.py b/swh/core/logger.py --- a/swh/core/logger.py +++ b/swh/core/logger.py @@ -5,11 +5,7 @@ import datetime import logging -import os -import socket -import psycopg2 -from psycopg2.extras import Json from systemd.journal import JournalHandler as _JournalHandler, send try: @@ -80,90 +76,6 @@ return str(value) -class PostgresHandler(logging.Handler): - """log handler that store messages in a Postgres DB - - See swh-core/swh/core/sql/log-schema.sql for the DB schema. - - All logging methods can be used as usual. Additionally, arbitrary metadata - can be passed to logging methods, requesting that they will be stored in - the DB as a single JSONB value. To do so, pass a dictionary to the 'extra' - kwarg of any logging method; all keys in that dictionary that start with - EXTRA_LOGDATA_PREFIX (currently: ``swh_``) will be extracted to form the - JSONB dictionary. The prefix will be stripped and not included in the DB. - - Note: the logger name will be used to fill the 'module' DB column. - - Sample usage:: - - logging.basicConfig(level=logging.INFO) - h = PostgresHandler('dbname=softwareheritage-log') - logging.getLogger().addHandler(h) - - logger.info('not so important notice', - extra={'swh_type': 'swh_logging_test', - 'swh_meditation': 'guru'}) - logger.warn('something weird just happened, did you see that?') - - """ - - def __init__(self, connstring): - """ - Create a Postgres log handler. - - Args: - config: configuration dictionary, with a key "log_db" containing a - libpq connection string to the log DB - """ - super().__init__() - - self.connstring = connstring - - self.fqdn = socket.getfqdn() # cache FQDN value - - def _connect(self): - return psycopg2.connect(self.connstring) - - def emit(self, record): - msg = self.format(record) - extra_data = get_extra_data(record) - - if 'task' in extra_data: - task_args = { - 'args': extra_data['task']['args'], - 'kwargs': extra_data['task']['kwargs'], - } - - try: - json_args = Json(task_args).getquoted() - except TypeError: - task_args = { - 'args': [''], - 'kwargs': {}, - } - else: - json_args_length = len(json_args) - if json_args_length >= 1000: - task_args = { - 'args': [''], - 'kwargs': {}, - } - - extra_data['task'].update(task_args) - - log_entry = (db_level_of_py_level(record.levelno), msg, - Json(extra_data), record.name, self.fqdn, - os.getpid()) - db = self._connect() - with db.cursor() as cur: - cur.execute('INSERT INTO log ' - '(level, message, data, src_module, src_host, src_pid)' - 'VALUES (%s, %s, %s, %s, %s, %s)', - log_entry) - db.commit() - db.close() - - class JournalHandler(_JournalHandler): def emit(self, record): """Write `record` as a journal event. diff --git a/swh/core/tests/test_logger.py b/swh/core/tests/test_logger.py --- a/swh/core/tests/test_logger.py +++ b/swh/core/tests/test_logger.py @@ -8,7 +8,7 @@ import pytest -from swh.core.logger import PostgresHandler +from swh.core.db.logger import PostgresHandler from swh.core.tests import SQL_DIR