diff --git a/PKG-INFO b/PKG-INFO index 03e4dc5..0dd8c9a 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.core -Version: 0.0.33 +Version: 0.0.34 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/debian/changelog b/debian/changelog index b6feacd..2c96986 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,231 +1,232 @@ -swh-core (0.0.33-1~swh1~bpo9+1) stretch-swh; urgency=medium +swh-core (0.0.34-1~swh1) unstable-swh; urgency=medium - * Rebuild for stretch-backports. + * Release swh.core v0.0.34 + * New modular database test fixture - -- Nicolas Dandrimont Mon, 19 Jun 2017 19:01:37 +0200 + -- Nicolas Dandrimont Mon, 07 Aug 2017 18:29:48 +0200 swh-core (0.0.33-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.33 * Be more conservative with remote API responses -- Nicolas Dandrimont Mon, 19 Jun 2017 19:01:38 +0200 swh-core (0.0.32-1~swh1) unstable-swh; urgency=medium * Release swh-core v0.0.32 * Add asynchronous streaming methods for internal APIs * Remove task arguments from systemd-journal loggers -- Nicolas Dandrimont Tue, 09 May 2017 14:04:22 +0200 swh-core (0.0.31-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.31 * Add explicit dependency on python3-systemd -- Nicolas Dandrimont Fri, 07 Apr 2017 15:11:26 +0200 swh-core (0.0.30-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.30 * drop swh.core.hashutil (moved to swh.model.hashutil) * add a systemd logger -- Nicolas Dandrimont Fri, 07 Apr 2017 11:49:15 +0200 swh-core (0.0.29-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.29 * Catch proper exception in the base API client -- Nicolas Dandrimont Thu, 02 Feb 2017 00:19:25 +0100 swh-core (0.0.28-1~swh1) unstable-swh; urgency=medium * v0.0.28 * Refactoring some common code into swh.core -- Antoine R. Dumont (@ardumont) Thu, 26 Jan 2017 14:54:22 +0100 swh-core (0.0.27-1~swh1) unstable-swh; urgency=medium * v0.0.27 * Fix issue with default boolean value -- Antoine R. Dumont (@ardumont) Thu, 20 Oct 2016 16:15:20 +0200 swh-core (0.0.26-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.26 * Raise an exception when a configuration file exists and is unreadable -- Nicolas Dandrimont Wed, 12 Oct 2016 10:16:09 +0200 swh-core (0.0.25-1~swh1) unstable-swh; urgency=medium * v0.0.25 * Add new function utils.cwd -- Antoine R. Dumont (@ardumont) Thu, 29 Sep 2016 21:29:37 +0200 swh-core (0.0.24-1~swh1) unstable-swh; urgency=medium * v0.0.24 * Deal with edge case in logger regarding json -- Antoine R. Dumont (@ardumont) Thu, 22 Sep 2016 12:21:09 +0200 swh-core (0.0.23-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.23 * Properly fix the PyYAML dependency -- Nicolas Dandrimont Tue, 23 Aug 2016 16:20:29 +0200 swh-core (0.0.22-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.22 * Proper loading of yaml and ini files in all paths -- Nicolas Dandrimont Fri, 19 Aug 2016 15:45:55 +0200 swh-core (0.0.21-1~swh1) unstable-swh; urgency=medium * v0.0.21 * Update test tools -- Antoine R. Dumont (@ardumont) Tue, 19 Jul 2016 14:47:01 +0200 swh-core (0.0.20-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.20 * Add some generic bytes <-> escaped unicode methods -- Nicolas Dandrimont Tue, 14 Jun 2016 16:54:41 +0200 swh-core (0.0.19-1~swh1) unstable-swh; urgency=medium * v0.0.19 * Resurrect swh.core.utils -- Antoine R. Dumont (@ardumont) Fri, 15 Apr 2016 12:40:43 +0200 swh-core (0.0.18-1~swh1) unstable-swh; urgency=medium * v0.0.18 * Add swh.core.utils * serializers: support UUIDs all around -- Antoine R. Dumont (@ardumont) Sat, 26 Mar 2016 11:16:33 +0100 swh-core (0.0.17-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.17 * Allow serialization of UUIDs -- Nicolas Dandrimont Fri, 04 Mar 2016 11:40:56 +0100 swh-core (0.0.16-1~swh1) unstable-swh; urgency=medium * Release swh.core version 0.0.16 * add bytehex_to_hash and hash_to_bytehex in hashutil * move scheduling utilities to swh.scheduler -- Nicolas Dandrimont Fri, 19 Feb 2016 18:12:10 +0100 swh-core (0.0.15-1~swh1) unstable-swh; urgency=medium * Release v0.0.15 * Add hashutil.hash_git_object -- Nicolas Dandrimont Wed, 16 Dec 2015 16:31:26 +0100 swh-core (0.0.14-1~swh1) unstable-swh; urgency=medium * v0.0.14 * Add simple README * Update license * swh.core.hashutil.hashfile can now deal with filepath as bytes -- Antoine R. Dumont (@ardumont) Fri, 23 Oct 2015 11:13:14 +0200 swh-core (0.0.13-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.13 -- Nicolas Dandrimont Fri, 09 Oct 2015 17:32:49 +0200 swh-core (0.0.12-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.12 -- Nicolas Dandrimont Tue, 06 Oct 2015 17:34:34 +0200 swh-core (0.0.11-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.11 -- Nicolas Dandrimont Sat, 03 Oct 2015 15:57:03 +0200 swh-core (0.0.10-1~swh1) unstable-swh; urgency=medium * Prepare deploying swh.core v0.0.10 -- Nicolas Dandrimont Sat, 03 Oct 2015 12:28:52 +0200 swh-core (0.0.9-1~swh1) unstable-swh; urgency=medium * Prepare deploying swh.core v0.0.9 -- Nicolas Dandrimont Sat, 03 Oct 2015 11:36:55 +0200 swh-core (0.0.8-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.8 -- Nicolas Dandrimont Thu, 01 Oct 2015 12:31:44 +0200 swh-core (0.0.7-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.7 -- Nicolas Dandrimont Thu, 01 Oct 2015 11:29:04 +0200 swh-core (0.0.6-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.6 -- Nicolas Dandrimont Tue, 29 Sep 2015 16:48:44 +0200 swh-core (0.0.5-1~swh1) unstable-swh; urgency=medium * Prepare v0.0.5 deployment -- Nicolas Dandrimont Tue, 29 Sep 2015 16:08:32 +0200 swh-core (0.0.4-1~swh1) unstable-swh; urgency=medium * Tagging swh.core 0.0.4 -- Nicolas Dandrimont Fri, 25 Sep 2015 15:41:26 +0200 swh-core (0.0.3-1~swh1) unstable-swh; urgency=medium * Tag swh.core v0.0.3 -- Nicolas Dandrimont Fri, 25 Sep 2015 11:07:10 +0200 swh-core (0.0.2-1~swh1) unstable-swh; urgency=medium * Deploy v0.0.2 -- Nicolas Dandrimont Wed, 23 Sep 2015 12:08:50 +0200 swh-core (0.0.1-1~swh1) unstable-swh; urgency=medium * Initial release * Tag v0.0.1 for deployment -- Nicolas Dandrimont Tue, 22 Sep 2015 14:52:26 +0200 diff --git a/swh.core.egg-info/PKG-INFO b/swh.core.egg-info/PKG-INFO index 03e4dc5..0dd8c9a 100644 --- a/swh.core.egg-info/PKG-INFO +++ b/swh.core.egg-info/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.core -Version: 0.0.33 +Version: 0.0.34 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/swh/core/tests/db_testing.py b/swh/core/tests/db_testing.py index 7895d00..d5ca12a 100644 --- a/swh/core/tests/db_testing.py +++ b/swh/core/tests/db_testing.py @@ -1,255 +1,262 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import psycopg2 import subprocess def pg_restore(dbname, dumpfile, dumptype='pg_dump'): """ Args: dbname: name of the DB to restore into dumpfile: path fo the dump file dumptype: one of 'pg_dump' (for binary dumps), 'psql' (for SQL dumps) """ assert dumptype in ['pg_dump', 'psql'] if dumptype == 'pg_dump': subprocess.check_call(['pg_restore', '--no-owner', '--no-privileges', '--dbname', dbname, dumpfile]) elif dumptype == 'psql': subprocess.check_call(['psql', '--quiet', '-f', dumpfile, dbname]) def pg_dump(dbname, dumpfile): subprocess.check_call(['pg_dump', '--no-owner', '--no-privileges', '-Fc', '-f', dumpfile, dbname]) def pg_dropdb(dbname): subprocess.check_call(['dropdb', dbname]) def pg_createdb(dbname): subprocess.check_call(['createdb', dbname]) def db_create(dbname, dump=None, dumptype='pg_dump'): """create the test DB and load the test data dump into it context: setUpClass """ try: pg_createdb(dbname) except subprocess.CalledProcessError: # try recovering once, in case pg_dropdb(dbname) # the db already existed pg_createdb(dbname) if dump: pg_restore(dbname, dump, dumptype) return dbname def db_destroy(dbname): """destroy the test DB context: tearDownClass """ pg_dropdb(dbname) def db_connect(dbname): """connect to the test DB and open a cursor context: setUp """ conn = psycopg2.connect('dbname=' + dbname) return { 'conn': conn, 'cursor': conn.cursor() } def db_close(conn): """rollback current transaction and disconnet from the test DB context: tearDown """ if not conn.closed: conn.rollback() conn.close() -class DbTestFixture(): +class DbTestConn: + def __init__(self, dbname): + self.dbname = dbname + + def __enter__(self): + self.db_setup = db_connect(self.dbname) + self.conn = self.db_setup['conn'] + self.cursor = self.db_setup['cursor'] + return self + + def __exit__(self, *_): + db_close(self.conn) + + +class DbTestContext: + def __init__(self, name='softwareheritage-test', dump=None, + dump_type='pg_dump'): + self.dbname = name + self.dump = dump + self.dump_type = dump_type + + def __enter__(self): + db_create(dbname=self.dbname, + dump=self.dump, + dumptype=self.dump_type) + return self + + def __exit__(self, *_): + db_destroy(self.dbname) + + +class DbTestFixture: """Mix this in a test subject class to get DB testing support. - The class can override the following class attributes: - TEST_DB_NAME: name of the DB used for testing - TEST_DB_DUMP: DB dump to be restored before running test methods; can - be set to None if no restore from dump is required - TEST_DB_DUMP_TYPE: one of 'pg_dump' (binary dump) or 'psql' (SQL dump) + Use the class method add_db() to add a new database to be tested. + Using this will create a DbTestConn entry in the `test_db` dictionary for + all the tests, indexed by the name of the database. - The test case class will then have the following attributes, accessible via - self: + Example: - dbname: name of the test database - conn: psycopg2 connection object - cursor: open psycopg2 cursor to the DB + class TestDb(DbTestFixture, unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.add_db('db_name', DUMP) + + def setUp(self): + db = self.test_db['db_name'] + print('conn: {}, cursor: {}'.format(db.conn, db.cursor)) To ensure test isolation, each test method of the test case class will execute in its own connection, cursor, and transaction. - To ensure setup/teardown methods are called, in case of multiple - inheritance DbTestFixture should be the first class in the inheritance - hierarchy. - Note that if you want to define setup/teardown methods, you need to explicitly call super() to ensure that the fixture setup/teardown methods are invoked. Here is an example where all setup/teardown methods are defined in a test case: class TestDb(DbTestFixture, unittest.TestCase): - @classmethod def setUpClass(cls): + # your add_db() calls here super().setUpClass() # your class setup code here def setUp(self): super().setUp() # your instance setup code here def tearDown(self): # your instance teardown code here super().tearDown() @classmethod def tearDownClass(cls): # your class teardown code here super().tearDownClass() """ - TEST_DB_NAME = 'softwareheritage-test' - TEST_DB_DUMP = None - TEST_DB_DUMP_TYPE = 'pg_dump' + _DB_DUMP_LIST = {} + _DB_LIST = {} + DB_TEST_FIXTURE_IMPORTED = True + + @classmethod + def add_db(cls, name='softwareheritage-test', dump=None, + dump_type='pg_dump'): + cls._DB_DUMP_LIST[name] = (dump, dump_type) @classmethod def setUpClass(cls): - cls.dbname = db_create(dbname=cls.TEST_DB_NAME, - dump=cls.TEST_DB_DUMP, - dumptype=cls.TEST_DB_DUMP_TYPE) + for name, (dump, dump_type) in cls._DB_DUMP_LIST.items(): + cls._DB_LIST[name] = DbTestContext(name, dump, dump_type) + cls._DB_LIST[name].__enter__() super().setUpClass() - def setUp(self): - db_setup = db_connect(self.dbname) - self.conn = db_setup['conn'] - self.cursor = db_setup['cursor'] - super().setUp() - - def tearDown(self): - super().tearDown() - db_close(self.conn) - @classmethod def tearDownClass(cls): super().tearDownClass() - db_destroy(cls.dbname) + for name, context in cls._DB_LIST.items(): + context.__exit__() + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.test_db = {} -class DbsTestFixture(): - """Mix this in a test subject class to get DB testing support with - multiple databases. + def setUp(self): + self.test_db = {} + for name in self._DB_LIST.keys(): + self.test_db[name] = DbTestConn(name) + self.test_db[name].__enter__() + super().setUp() - The class can override the following class attributes: - TEST_DB_NAMES: names of the DB used for testing - TEST_DB_DUMPS: DB dumps to be restored before running test methods; can - be set to [] if no restore from dump is required - TEST_DB_DUMP_TYPES: List of one of 'pg_dump' (binary dump) or 'psql' - (SQL dump) + def tearDown(self): + super().tearDown() + for name in self._DB_LIST.keys(): + self.test_db[name].__exit__() - The test case class will then have the following attributes, accessible via - self: + def reset_db_tables(self, name, excluded=None): + db = self.test_db[name] + conn = db.conn + cursor = db.cursor - dbnames: name of the test database - conns: psycopg2 connection object - cursors: open psycopg2 cursor to the DB + cursor.execute("""SELECT table_name FROM information_schema.tables + WHERE table_schema = %s""", ('public',)) - To ensure test isolation, each test method of the test case class will - execute in its own connection, cursor, and transaction. + tables = set(table for (table,) in cursor.fetchall()) + if excluded is not None: + tables -= set(excluded) - To ensure setup/teardown methods are called, in case of multiple - inheritance DbTestFixture should be the first class in the inheritance - hierarchy. + for table in tables: + cursor.execute('truncate table %s cascade' % table) - Note that if you want to define setup/teardown methods, you need to - explicitly call super() to ensure that the fixture setup/teardown methods - are invoked. Here is an example where all setup/teardown methods are - defined in a test case: + conn.commit() - class TestDb(DbTestFixture, unittest.TestCase): - @classmethod - def setUpClass(cls): - super().setUpClass() - # your class setup code here +class SingleDbTestFixture(DbTestFixture): + """Simplified fixture like DbTest but that can only handle a single DB. - def setUp(self): - super().setUp() - # your instance setup code here + Gives access to shortcuts like self.cursor and self.conn. - def tearDown(self): - # your instance teardown code here - super().tearDown() + DO NOT use this with other fixtures that need to access databases, like + StorageTestFixture. - @classmethod - def tearDownClass(cls): - # your class teardown code here - super().tearDownClass() + The class can override the following class attributes: + TEST_DB_NAME: name of the DB used for testing + TEST_DB_DUMP: DB dump to be restored before running test methods; can + be set to None if no restore from dump is required + TEST_DB_DUMP_TYPE: one of 'pg_dump' (binary dump) or 'psql' (SQL dump) + The test case class will then have the following attributes, accessible via + self: + + dbname: name of the test database + conn: psycopg2 connection object + cursor: open psycopg2 cursor to the DB """ - TEST_DB_NAMES = ['softwareheritage-test'] - TEST_DB_DUMPS = [] - TEST_DB_DUMP_TYPES = ['pg_dump'] + TEST_DB_NAME = 'softwareheritage-test' + TEST_DB_DUMP = None + TEST_DB_DUMP_TYPE = 'pg_dump' @classmethod def setUpClass(cls): - dbnames = [] - for i, dbname in enumerate(cls.TEST_DB_NAMES): - try: - dbname = db_create(dbname, - dump=cls.TEST_DB_DUMPS[i], - dumptype=cls.TEST_DB_DUMP_TYPES[i]) - finally: - dbnames.append(dbname) - - cls.dbnames = dbnames + cls.dbname = cls.TEST_DB_NAME + cls.add_db(name=cls.TEST_DB_NAME, + dump=cls.TEST_DB_DUMP, + dump_type=cls.TEST_DB_DUMP_TYPE) super().setUpClass() def setUp(self): - conns = [] - cursors = [] - for i, dbname in enumerate(self.dbnames): - db_setup = db_connect(dbname) - conns.append(db_setup['conn']) - cursors.append(db_setup['cursor']) - - self.conns = conns - self.cursors = cursors super().setUp() - def tearDown(self): - super().tearDown() - for conn in self.conns: - db_close(conn) - - @classmethod - def tearDownClass(cls): - super().tearDownClass() - for dbname in cls.dbnames: - db_destroy(dbname) + db = self.test_db[self.TEST_DB_NAME] + self.conn = db.conn + self.cursor = db.cursor diff --git a/swh/core/tests/test_logger.py b/swh/core/tests/test_logger.py index 4c16b27..0547873 100644 --- a/swh/core/tests/test_logger.py +++ b/swh/core/tests/test_logger.py @@ -1,50 +1,50 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import os import unittest from nose.tools import istest from nose.plugins.attrib import attr from swh.core.logger import PostgresHandler -from swh.core.tests.db_testing import DbTestFixture +from swh.core.tests.db_testing import SingleDbTestFixture TEST_DIR = os.path.dirname(os.path.abspath(__file__)) SQL_DIR = os.path.join(TEST_DIR, '../../../sql') @attr('db') -class PgLogHandler(DbTestFixture, unittest.TestCase): +class PgLogHandler(SingleDbTestFixture, unittest.TestCase): TEST_DB_DUMP = os.path.join(SQL_DIR, 'log-schema.sql') TEST_DB_DUMP_TYPE = 'psql' def setUp(self): super().setUp() self.modname = 'swh.core.tests.test_logger' self.logger = logging.Logger(self.modname, logging.DEBUG) self.logger.addHandler(PostgresHandler('dbname=' + self.TEST_DB_NAME)) def tearDown(self): logging.shutdown() super().tearDown() @istest def log(self): self.logger.info('notice', extra={'swh_type': 'test entry', 'swh_data': 42}) self.logger.warn('warning') with self.conn.cursor() as cur: cur.execute('SELECT level, message, data, src_module FROM log') db_log_entries = cur.fetchall() self.assertIn(('info', 'notice', {'type': 'test entry', 'data': 42}, self.modname), db_log_entries) self.assertIn(('warning', 'warning', {}, self.modname), db_log_entries) diff --git a/version.txt b/version.txt index b914bef..b3dbbaa 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.33-0-gcb1e915 \ No newline at end of file +v0.0.34-0-g1c0167c \ No newline at end of file