Page MenuHomeSoftware Heritage
Files F9341654

No OneTemporary
Actions

View Options

diff --git a/PKG-INFO b/PKG-INFO
index a502622..a3299d1 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,28 +1,28 @@
Metadata-Version: 2.1
Name: swh.core
-Version: 0.0.47
+Version: 0.0.48
Summary: Software Heritage core utilities
Home-page: https://forge.softwareheritage.org/diffusion/DCORE/
Author: Software Heritage developers
Author-email: swh-devel@inria.fr
License: UNKNOWN
Project-URL: Funding, https://www.softwareheritage.org/donate
-Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest
Project-URL: Source, https://forge.softwareheritage.org/source/swh-core
+Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest
Description: swh-core
========
core library for swh's modules:
- config parser
- hash computations
- serialization
- logging mechanism
Platform: UNKNOWN
Classifier: Programming Language :: Python :: 3
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
Classifier: Operating System :: OS Independent
Classifier: Development Status :: 5 - Production/Stable
Description-Content-Type: text/markdown
Provides-Extra: testing
diff --git a/swh.core.egg-info/PKG-INFO b/swh.core.egg-info/PKG-INFO
index a502622..a3299d1 100644
--- a/swh.core.egg-info/PKG-INFO
+++ b/swh.core.egg-info/PKG-INFO
@@ -1,28 +1,28 @@
Metadata-Version: 2.1
Name: swh.core
-Version: 0.0.47
+Version: 0.0.48
Summary: Software Heritage core utilities
Home-page: https://forge.softwareheritage.org/diffusion/DCORE/
Author: Software Heritage developers
Author-email: swh-devel@inria.fr
License: UNKNOWN
Project-URL: Funding, https://www.softwareheritage.org/donate
-Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest
Project-URL: Source, https://forge.softwareheritage.org/source/swh-core
+Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest
Description: swh-core
========
core library for swh's modules:
- config parser
- hash computations
- serialization
- logging mechanism
Platform: UNKNOWN
Classifier: Programming Language :: Python :: 3
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
Classifier: Operating System :: OS Independent
Classifier: Development Status :: 5 - Production/Stable
Description-Content-Type: text/markdown
Provides-Extra: testing
diff --git a/swh/core/cli.py b/swh/core/cli.py
index c8c7684..0a63ceb 100755
--- a/swh/core/cli.py
+++ b/swh/core/cli.py
@@ -1,71 +1,80 @@
#!/usr/bin/env python3
# Copyright (C) 2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import warnings
warnings.filterwarnings("ignore") # noqa prevent psycopg from telling us sh*t
from os import path
import glob
import click
from importlib import import_module
from swh.core.utils import numfile_sortkey as sortkey
-from swh.core.tests.db_testing import pg_createdb, pg_restore, DB_DUMP_TYPES
+from swh.core.tests.db_testing import (
+ pg_createdb, pg_restore, DB_DUMP_TYPES,
+ swh_db_version
+)
@click.command()
@click.argument('module', nargs=-1, required=True)
@click.option('--db-name', '-d', help='Database name.',
default='softwareheritage-dev', show_default=True)
-@click.option('--create/--no-create', '-c',
- help='Attempt to create the database', default=True)
-def db_init(module, db_name=None, create=True):
+def db_init(module, db_name=None):
"""Initialise a database for the Software Heritage <module>. By
default, attempts to create the database first.
Example:
swh-db-init storage -d swh-test
If you want to specify non-default postgresql connection parameters,
please provide them using standard environment variables.
See psql(1) man page (section ENVIRONMENTS) for details.
Example:
PGPORT=5434 swh-db-init indexer -d swh-indexer
"""
-
dump_files = []
for modname in module:
if not modname.startswith('swh.'):
modname = 'swh.{}'.format(modname)
try:
m = import_module(modname)
except ImportError:
raise click.BadParameter(
'Unable to load module {}'.format(modname))
sqldir = path.join(path.dirname(m.__file__), 'sql')
if not path.isdir(sqldir):
raise click.BadParameter(
'Module {} does not provide a db schema '
'(no sql/ dir)'.format(modname))
dump_files.extend(sorted(glob.glob(path.join(sqldir, '*.sql')),
key=sortkey))
- if create:
- pg_createdb(db_name)
- dump_files = [(x, DB_DUMP_TYPES[path.splitext(x)[1]])
- for x in dump_files]
- for dump, dtype in dump_files:
- click.secho('Loading {}'.format(dump), fg='yellow')
- pg_restore(db_name, dump, dtype)
+ # Create the db (or fail silently if already existing)
+ pg_createdb(db_name, check=False)
+ # Try to retrieve the db version if any
+ db_version = swh_db_version(db_name)
+ if not db_version: # Initialize the db
+ dump_files = [(x, DB_DUMP_TYPES[path.splitext(x)[1]])
+ for x in dump_files]
+ for dump, dtype in dump_files:
+ click.secho('Loading {}'.format(dump), fg='yellow')
+ pg_restore(db_name, dump, dtype)
+
+ db_version = swh_db_version(db_name)
+
+ # TODO: Ideally migrate the version from db_version to the latest
+ # db version
- click.secho('DONE database is {}'.format(db_name), fg='green', bold=True)
+ click.secho('DONE database is {} version {}'.format(db_name, db_version),
+ fg='green', bold=True)
diff --git a/swh/core/tests/db_testing.py b/swh/core/tests/db_testing.py
index a0ae407..10f0d86 100644
--- a/swh/core/tests/db_testing.py
+++ b/swh/core/tests/db_testing.py
@@ -1,281 +1,314 @@
-# Copyright (C) 2015 The Software Heritage developers
+# Copyright (C) 2015-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import os
import glob
import psycopg2
import subprocess
from swh.core.utils import numfile_sortkey as sortkey
DB_DUMP_TYPES = {'.sql': 'psql', '.dump': 'pg_dump'}
+def swh_db_version(dbname_or_service):
+ """Retrieve the swh version if any. In case of the db not initialized,
+ this returns None. Otherwise, this returns the db's version.
+
+ Args:
+ dbname_or_service (str): The db's name or service
+
+ Returns:
+ Optional[Int]: Either the db's version or None
+
+ """
+ query = 'select version from dbversion order by dbversion desc limit 1'
+ cmd = [
+ 'psql', '--tuples-only', '--no-psqlrc', '--quiet',
+ '-v', 'ON_ERROR_STOP=1', "--command=%s" % query,
+ dbname_or_service
+ ]
+
+ try:
+ r = subprocess.run(cmd, check=True, stdout=subprocess.PIPE,
+ universal_newlines=True)
+ result = int(r.stdout.strip())
+ except Exception: # db not initialized
+ result = None
+ return result
+
+
def pg_restore(dbname, dumpfile, dumptype='pg_dump'):
"""
Args:
dbname: name of the DB to restore into
dumpfile: path fo the dump file
dumptype: one of 'pg_dump' (for binary dumps), 'psql' (for SQL dumps)
"""
assert dumptype in ['pg_dump', 'psql']
if dumptype == 'pg_dump':
subprocess.check_call(['pg_restore', '--no-owner', '--no-privileges',
'--dbname', dbname, dumpfile])
elif dumptype == 'psql':
subprocess.check_call(['psql', '--quiet',
'--no-psqlrc',
'-v', 'ON_ERROR_STOP=1',
'-f', dumpfile,
dbname])
def pg_dump(dbname, dumpfile):
subprocess.check_call(['pg_dump', '--no-owner', '--no-privileges', '-Fc',
'-f', dumpfile, dbname])
def pg_dropdb(dbname):
subprocess.check_call(['dropdb', dbname])
-def pg_createdb(dbname):
- subprocess.check_call(['createdb', dbname])
+def pg_createdb(dbname, check=True):
+ """Create a db. If check is True and the db already exists, this will
+ raise an exception (original behavior). If check is False and
+ the db already exists, this will fail silently. If the db does
+ not exist, the db will be created.
+
+ """
+ subprocess.run(['createdb', dbname], check=check)
def db_create(dbname, dumps=None):
"""create the test DB and load the test data dumps into it
dumps is an iterable of couples (dump_file, dump_type).
context: setUpClass
"""
try:
pg_createdb(dbname)
except subprocess.CalledProcessError: # try recovering once, in case
pg_dropdb(dbname) # the db already existed
pg_createdb(dbname)
for dump, dtype in dumps:
pg_restore(dbname, dump, dtype)
return dbname
def db_destroy(dbname):
"""destroy the test DB
context: tearDownClass
"""
pg_dropdb(dbname)
def db_connect(dbname):
"""connect to the test DB and open a cursor
context: setUp
"""
conn = psycopg2.connect('dbname=' + dbname)
return {
'conn': conn,
'cursor': conn.cursor()
}
def db_close(conn):
"""rollback current transaction and disconnect from the test DB
context: tearDown
"""
if not conn.closed:
conn.rollback()
conn.close()
class DbTestConn:
def __init__(self, dbname):
self.dbname = dbname
def __enter__(self):
self.db_setup = db_connect(self.dbname)
self.conn = self.db_setup['conn']
self.cursor = self.db_setup['cursor']
return self
def __exit__(self, *_):
db_close(self.conn)
class DbTestContext:
def __init__(self, name='softwareheritage-test', dumps=None):
self.dbname = name
self.dumps = dumps
def __enter__(self):
db_create(dbname=self.dbname,
dumps=self.dumps)
return self
def __exit__(self, *_):
db_destroy(self.dbname)
class DbTestFixture:
"""Mix this in a test subject class to get DB testing support.
Use the class method add_db() to add a new database to be tested.
Using this will create a DbTestConn entry in the `test_db` dictionary for
all the tests, indexed by the name of the database.
Example:
class TestDb(DbTestFixture, unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.add_db('db_name', DUMP)
super().setUpClass()
def setUp(self):
db = self.test_db['db_name']
print('conn: {}, cursor: {}'.format(db.conn, db.cursor))
To ensure test isolation, each test method of the test case class will
execute in its own connection, cursor, and transaction.
Note that if you want to define setup/teardown methods, you need to
explicitly call super() to ensure that the fixture setup/teardown methods
are invoked. Here is an example where all setup/teardown methods are
defined in a test case:
class TestDb(DbTestFixture, unittest.TestCase):
@classmethod
def setUpClass(cls):
# your add_db() calls here
super().setUpClass()
# your class setup code here
def setUp(self):
super().setUp()
# your instance setup code here
def tearDown(self):
# your instance teardown code here
super().tearDown()
@classmethod
def tearDownClass(cls):
# your class teardown code here
super().tearDownClass()
"""
_DB_DUMP_LIST = {}
_DB_LIST = {}
DB_TEST_FIXTURE_IMPORTED = True
@classmethod
def add_db(cls, name='softwareheritage-test', dumps=None):
cls._DB_DUMP_LIST[name] = dumps
@classmethod
def setUpClass(cls):
for name, dumps in cls._DB_DUMP_LIST.items():
cls._DB_LIST[name] = DbTestContext(name, dumps)
cls._DB_LIST[name].__enter__()
super().setUpClass()
@classmethod
def tearDownClass(cls):
super().tearDownClass()
for name, context in cls._DB_LIST.items():
context.__exit__()
def setUp(self):
self.test_db = {}
for name in self._DB_LIST.keys():
self.test_db[name] = DbTestConn(name)
self.test_db[name].__enter__()
super().setUp()
def tearDown(self):
super().tearDown()
for name in self._DB_LIST.keys():
self.test_db[name].__exit__()
def reset_db_tables(self, name, excluded=None):
db = self.test_db[name]
conn = db.conn
cursor = db.cursor
cursor.execute("""SELECT table_name FROM information_schema.tables
WHERE table_schema = %s""", ('public',))
tables = set(table for (table,) in cursor.fetchall())
if excluded is not None:
tables -= set(excluded)
for table in tables:
cursor.execute('truncate table %s cascade' % table)
conn.commit()
class SingleDbTestFixture(DbTestFixture):
"""Simplified fixture like DbTest but that can only handle a single DB.
Gives access to shortcuts like self.cursor and self.conn.
DO NOT use this with other fixtures that need to access databases, like
StorageTestFixture.
The class can override the following class attributes:
TEST_DB_NAME: name of the DB used for testing
TEST_DB_DUMP: DB dump to be restored before running test methods; can
be set to None if no restore from dump is required.
If the dump file name endswith"
- '.sql' it will be loaded via psql,
- '.dump' it will be loaded via pg_restore.
Other file extensions will be ignored.
Can be a string or a list of strings; each path will be expanded
using glob pattern matching.
The test case class will then have the following attributes, accessible via
self:
dbname: name of the test database
conn: psycopg2 connection object
cursor: open psycopg2 cursor to the DB
"""
TEST_DB_NAME = 'softwareheritage-test'
TEST_DB_DUMP = None
@classmethod
def setUpClass(cls):
cls.dbname = cls.TEST_DB_NAME # XXX to kill?
dump_files = cls.TEST_DB_DUMP
if isinstance(dump_files, str):
dump_files = [dump_files]
all_dump_files = []
for files in dump_files:
all_dump_files.extend(
sorted(glob.glob(files), key=sortkey))
all_dump_files = [(x, DB_DUMP_TYPES[os.path.splitext(x)[1]])
for x in all_dump_files]
cls.add_db(name=cls.TEST_DB_NAME,
dumps=all_dump_files)
super().setUpClass()
def setUp(self):
super().setUp()
db = self.test_db[self.TEST_DB_NAME]
self.conn = db.conn
self.cursor = db.cursor
diff --git a/version.txt b/version.txt
index b01633f..fdc395d 100644
--- a/version.txt
+++ b/version.txt
@@ -1 +1 @@
-v0.0.47-0-g386f8e2
\ No newline at end of file
+v0.0.48-0-g36b28c1
\ No newline at end of file

File Metadata

Mime Type
text/x-diff
Expires
Fri, Jul 4, 12:13 PM (2 w, 5 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3215279

Event Timeline

About · Developer information · Legal