diff --git a/requirements-swh-journal.txt b/requirements-swh-journal.txt index 724133a4..e43a37f4 100644 --- a/requirements-swh-journal.txt +++ b/requirements-swh-journal.txt @@ -1 +1 @@ -swh.journal >= 0.0.17 +swh.journal >= 0.0.30 diff --git a/swh/storage/tests/conftest.py b/swh/storage/tests/conftest.py index 6a2c341f..6cc476f0 100644 --- a/swh/storage/tests/conftest.py +++ b/swh/storage/tests/conftest.py @@ -1,276 +1,254 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import glob import pytest from typing import Union -from unittest.mock import patch from pytest_postgresql import factories from pytest_postgresql.janitor import DatabaseJanitor, psycopg2, Version from os import path, environ from hypothesis import settings from typing import Dict import swh.storage from swh.core.utils import numfile_sortkey as sortkey from swh.model.tests.generate_testdata import gen_contents, gen_origins from swh.model.model import ( Content, Directory, Origin, OriginVisit, Release, Revision, SkippedContent, Snapshot, ) -from swh.journal.writer.inmemory import InMemoryJournalWriter OBJECT_FACTORY = { "content": Content.from_dict, "directory": Directory.from_dict, "origin": Origin.from_dict, "origin_visit": OriginVisit.from_dict, "release": Release.from_dict, "revision": Revision.from_dict, "skipped_content": SkippedContent.from_dict, "snapshot": Snapshot.from_dict, } SQL_DIR = path.join(path.dirname(swh.storage.__file__), "sql") environ["LC_ALL"] = "C.UTF-8" DUMP_FILES = path.join(SQL_DIR, "*.sql") # define tests profile. Full documentation is at: # https://hypothesis.readthedocs.io/en/latest/settings.html#settings-profiles settings.register_profile("fast", max_examples=5, deadline=5000) settings.register_profile("slow", max_examples=20, deadline=5000) @pytest.fixture def swh_storage_backend_config(postgresql_proc, swh_storage_postgresql): yield { "cls": "local", "db": "postgresql://{user}@{host}:{port}/{dbname}".format( host=postgresql_proc.host, port=postgresql_proc.port, user="postgres", dbname="tests", ), "objstorage": {"cls": "memory", "args": {}}, "journal_writer": {"cls": "memory",}, } -class BWCompatInMemoryJournalWriter(InMemoryJournalWriter): - """InMemoryJournalWriter that enforces conversion of objects to model entities - - This is required until swh.journal 0.0.30 is available - """ - - def write_addition(self, object_type, object_): - if isinstance(object_, dict): - object_ = OBJECT_FACTORY[object_type](object_) - self.objects.append((object_type, object_)) - - write_update = write_addition - - @pytest.fixture def swh_storage(swh_storage_backend_config): - storage_config = {"cls": "validate", "storage": swh_storage_backend_config} - with patch( - "swh.journal.writer.inmemory.InMemoryJournalWriter", - return_value=BWCompatInMemoryJournalWriter(), - ): - storage = swh.storage.get_storage(**storage_config) - return storage + return swh.storage.get_storage(cls="validate", storage=swh_storage_backend_config) @pytest.fixture def swh_contents(swh_storage): contents = gen_contents(n=20) swh_storage.content_add([c for c in contents if c["status"] != "absent"]) swh_storage.skipped_content_add([c for c in contents if c["status"] == "absent"]) return contents @pytest.fixture def swh_origins(swh_storage): origins = gen_origins(n=100) swh_storage.origin_add(origins) return origins # the postgres_fact factory fixture below is mostly a copy of the code # from pytest-postgresql. We need a custom version here to be able to # specify our version of the DBJanitor we use. def postgresql_fact(process_fixture_name, db_name=None, dump_files=DUMP_FILES): @pytest.fixture def postgresql_factory(request): """ Fixture factory for PostgreSQL. :param FixtureRequest request: fixture request object :rtype: psycopg2.connection :returns: postgresql client """ config = factories.get_config(request) if not psycopg2: raise ImportError("No module named psycopg2. Please install it.") proc_fixture = request.getfixturevalue(process_fixture_name) # _, config = try_import('psycopg2', request) pg_host = proc_fixture.host pg_port = proc_fixture.port pg_user = proc_fixture.user pg_options = proc_fixture.options pg_db = db_name or config["dbname"] with SwhDatabaseJanitor( pg_user, pg_host, pg_port, pg_db, proc_fixture.version, dump_files=dump_files, ): connection = psycopg2.connect( dbname=pg_db, user=pg_user, host=pg_host, port=pg_port, options=pg_options, ) yield connection connection.close() return postgresql_factory swh_storage_postgresql = postgresql_fact("postgresql_proc") # This version of the DatabaseJanitor implement a different setup/teardown # behavior than than the stock one: instead of dropping, creating and # initializing the database for each test, it create and initialize the db only # once, then it truncate the tables. This is needed to have acceptable test # performances. class SwhDatabaseJanitor(DatabaseJanitor): def __init__( self, user: str, host: str, port: str, db_name: str, version: Union[str, float, Version], dump_files: str = DUMP_FILES, ) -> None: super().__init__(user, host, port, db_name, version) self.dump_files = sorted(glob.glob(dump_files), key=sortkey) def db_setup(self): with psycopg2.connect( dbname=self.db_name, user=self.user, host=self.host, port=self.port, ) as cnx: with cnx.cursor() as cur: for fname in self.dump_files: with open(fname) as fobj: sql = fobj.read().replace("concurrently", "").strip() if sql: cur.execute(sql) cnx.commit() def db_reset(self): with psycopg2.connect( dbname=self.db_name, user=self.user, host=self.host, port=self.port, ) as cnx: with cnx.cursor() as cur: cur.execute( "SELECT table_name FROM information_schema.tables " "WHERE table_schema = %s", ("public",), ) tables = set(table for (table,) in cur.fetchall()) for table in tables: cur.execute("truncate table %s cascade" % table) cur.execute( "SELECT sequence_name FROM information_schema.sequences " "WHERE sequence_schema = %s", ("public",), ) seqs = set(seq for (seq,) in cur.fetchall()) for seq in seqs: cur.execute("ALTER SEQUENCE %s RESTART;" % seq) cnx.commit() def init(self): with self.cursor() as cur: cur.execute( "SELECT COUNT(1) FROM pg_database WHERE datname=%s;", (self.db_name,) ) db_exists = cur.fetchone()[0] == 1 if db_exists: cur.execute( "UPDATE pg_database SET datallowconn=true " "WHERE datname = %s;", (self.db_name,), ) if db_exists: self.db_reset() else: with self.cursor() as cur: cur.execute('CREATE DATABASE "{}";'.format(self.db_name)) self.db_setup() def drop(self): pid_column = "pid" with self.cursor() as cur: cur.execute( "UPDATE pg_database SET datallowconn=false " "WHERE datname = %s;", (self.db_name,), ) cur.execute( "SELECT pg_terminate_backend(pg_stat_activity.{})" "FROM pg_stat_activity " "WHERE pg_stat_activity.datname = %s;".format(pid_column), (self.db_name,), ) @pytest.fixture def sample_data() -> Dict: """Pre-defined sample storage object data to manipulate Returns: Dict of data (keys: content, directory, revision, release, person, origin) """ from .storage_data import data return { "content": [data.cont, data.cont2], "content_metadata": [data.cont3], "skipped_content": [data.skipped_cont, data.skipped_cont2], "person": [data.person], "directory": [data.dir2, data.dir], "revision": [data.revision, data.revision2, data.revision3], "release": [data.release, data.release2, data.release3], "snapshot": [data.snapshot], "origin": [data.origin, data.origin2], "tool": [data.metadata_tool], "provider": [data.provider], "origin_metadata": [data.origin_metadata, data.origin_metadata2], } diff --git a/swh/storage/tests/test_api_client.py b/swh/storage/tests/test_api_client.py index 7012110f..16a065f9 100644 --- a/swh/storage/tests/test_api_client.py +++ b/swh/storage/tests/test_api_client.py @@ -1,74 +1,67 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from unittest.mock import patch import pytest import swh.storage.api.server as server import swh.storage.storage from swh.storage import get_storage from swh.storage.tests.test_storage import TestStorageGeneratedData # noqa from swh.storage.tests.test_storage import TestStorage as _TestStorage -from swh.storage.tests.conftest import BWCompatInMemoryJournalWriter # tests are executed using imported classes (TestStorage and # TestStorageGeneratedData) using overloaded swh_storage fixture # below @pytest.fixture def app_server(): - storage_config = { - "cls": "memory", - "journal_writer": {"cls": "memory",}, - } - with patch( - "swh.journal.writer.inmemory.InMemoryJournalWriter", - return_value=BWCompatInMemoryJournalWriter(), - ): - server.storage = swh.storage.get_storage(**storage_config) + server.storage = swh.storage.get_storage( + cls="memory", journal_writer={"cls": "memory"} + ) yield server @pytest.fixture def app(app_server): return app_server.app @pytest.fixture def swh_rpc_client_class(): def storage_factory(**kwargs): storage_config = {"cls": "validate", "storage": {"cls": "remote", **kwargs,}} return get_storage(**storage_config) return storage_factory @pytest.fixture def swh_storage(swh_rpc_client, app_server): # This version of the swh_storage fixture uses the swh_rpc_client fixture # to instantiate a RemoteStorage (see swh_rpc_client_class above) that # proxies, via the swh.core RPC mechanism, the local (in memory) storage # configured in the app_server fixture above. # # Also note that, for the sake of # making it easier to write tests, the in-memory journal writer of the # in-memory backend storage is attached to the RemoteStorage as its # journal_writer attribute. storage = swh_rpc_client journal_writer = getattr(storage, "journal_writer", None) storage.journal_writer = app_server.storage.journal_writer yield storage storage.journal_writer = journal_writer class TestStorage(_TestStorage): def test_content_update(self, swh_storage, app_server): # TODO, journal_writer not supported swh_storage.journal_writer.journal = None with patch.object(server.storage.journal_writer, "journal", None): super().test_content_update(swh_storage) diff --git a/swh/storage/tests/test_api_client_dicts.py b/swh/storage/tests/test_api_client_dicts.py index c5704f0b..a59ab7af 100644 --- a/swh/storage/tests/test_api_client_dicts.py +++ b/swh/storage/tests/test_api_client_dicts.py @@ -1,56 +1,51 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from unittest.mock import patch import pytest from swh.storage.api.client import RemoteStorage import swh.storage.api.server as server import swh.storage.storage from swh.storage.tests.test_storage import TestStorageGeneratedData # noqa from swh.storage.tests.test_storage import TestStorage as _TestStorage -from swh.storage.tests.conftest import BWCompatInMemoryJournalWriter from swh.storage.tests.test_api_client import swh_storage # noqa # tests are executed using imported classes (TestStorage and # TestStorageGeneratedData) using overloaded swh_storage fixture # below @pytest.fixture def app_server(): storage_config = { "cls": "validate", "storage": {"cls": "memory", "journal_writer": {"cls": "memory",},}, } - with patch( - "swh.journal.writer.inmemory.InMemoryJournalWriter", - return_value=BWCompatInMemoryJournalWriter(), - ): - server.storage = swh.storage.get_storage(**storage_config) + server.storage = swh.storage.get_storage(**storage_config) yield server @pytest.fixture def app(app_server): return app_server.app @pytest.fixture def swh_rpc_client_class(): return RemoteStorage class TestStorage(_TestStorage): def test_content_update(self, swh_storage, app_server): # noqa # TODO, journal_writer not supported swh_storage.journal_writer.journal = None with patch.object(server.storage.journal_writer, "journal", None): super().test_content_update(swh_storage) @pytest.mark.skip("non-applicable test") def test_content_add_from_lazy_content(self): pass