Changeset View
Standalone View
swh/provenance/tests/conftest.py
# Copyright (C) 2021 The Software Heritage developers | # Copyright (C) 2021 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from datetime import datetime, timedelta, timezone | from datetime import datetime, timedelta, timezone | ||||
from os import path | from os import path | ||||
from typing import Any, Dict, Iterable, Iterator | from typing import Any, Dict, Iterable, Iterator | ||||
from _pytest.fixtures import SubRequest | from _pytest.fixtures import SubRequest | ||||
import msgpack | import msgpack | ||||
import psycopg2.extensions | import psycopg2.extensions | ||||
import pytest | import pytest | ||||
from unittest.mock import patch | |||||
from pytest_postgresql.factories import postgresql | from pytest_postgresql.factories import postgresql | ||||
from swh.journal.serializers import msgpack_ext_hook | from swh.journal.serializers import msgpack_ext_hook | ||||
from swh.provenance import get_provenance, get_provenance_storage | from swh.provenance import get_provenance, get_provenance_storage | ||||
from swh.provenance.api.client import RemoteProvenanceStorage | from swh.provenance.api.client import RemoteProvenanceStorage | ||||
import swh.provenance.api.server as server | import swh.provenance.api.server as server | ||||
from swh.provenance.archive import ArchiveInterface | from swh.provenance.archive import ArchiveInterface | ||||
from swh.provenance.interface import ProvenanceInterface, ProvenanceStorageInterface | from swh.provenance.interface import ProvenanceInterface, ProvenanceStorageInterface | ||||
from swh.provenance.storage.archive import ArchiveStorage | from swh.provenance.storage.archive import ArchiveStorage | ||||
from swh.storage.interface import StorageInterface | from swh.storage.interface import StorageInterface | ||||
from swh.storage.replay import process_replay_objects | from swh.storage.replay import process_replay_objects | ||||
import mongomock | |||||
aeviso: I believe this include doesn't belong in this group but to the previous one. Pre-commit hooks… | |||||
@pytest.fixture( | @pytest.fixture( | ||||
params=[ | params=[ | ||||
"with-path", | "with-path", | ||||
"without-path", | "without-path", | ||||
"with-path-denormalized", | "with-path-denormalized", | ||||
"without-path-denormalized", | "without-path-denormalized", | ||||
] | ] | ||||
) | ) | ||||
def populated_db( | def provenance_postgresqldb( | ||||
request: SubRequest, | request: SubRequest, | ||||
postgresql: psycopg2.extensions.connection, | postgresql: psycopg2.extensions.connection, | ||||
) -> Dict[str, str]: | ) -> Dict[str, str]: | ||||
"""return a working and initialized provenance db""" | """return a working and initialized provenance db""" | ||||
from swh.core.cli.db import populate_database_for_package | from swh.core.cli.db import populate_database_for_package | ||||
populate_database_for_package( | populate_database_for_package( | ||||
"swh.provenance", postgresql.dsn, flavor=request.param | "swh.provenance", postgresql.dsn, flavor=request.param | ||||
) | ) | ||||
return postgresql.get_dsn_parameters() | return postgresql.get_dsn_parameters() | ||||
# the Flask app used as server in these tests | # the Flask app used as server in these tests | ||||
@pytest.fixture | @pytest.fixture | ||||
def app(populated_db: Dict[str, str]) -> Iterator[server.ProvenanceStorageServerApp]: | def app( | ||||
provenance_postgresqldb: Dict[str, str] | |||||
) -> Iterator[server.ProvenanceStorageServerApp]: | |||||
assert hasattr(server, "storage") | assert hasattr(server, "storage") | ||||
server.storage = get_provenance_storage(cls="postgresql", db=populated_db) | server.storage = get_provenance_storage( | ||||
cls="postgresql", db=provenance_postgresqldb | |||||
) | |||||
yield server.app | yield server.app | ||||
# the RPCClient class used as client used in these tests | # the RPCClient class used as client used in these tests | ||||
@pytest.fixture | @pytest.fixture | ||||
def swh_rpc_client_class() -> type: | def swh_rpc_client_class() -> type: | ||||
return RemoteProvenanceStorage | return RemoteProvenanceStorage | ||||
@pytest.fixture(params=["postgresql", "remote"]) | @pytest.fixture(params=["mongodb"]) | ||||
def provenance_storage( | def provenance_storage( | ||||
request: SubRequest, | request: SubRequest, | ||||
populated_db: Dict[str, str], | provenance_postgresqldb: Dict[str, str], | ||||
Not Done Inline ActionsNot sure how to improve this to avoid repeating the test when a fixture that is not used gets a new instantiation. For instance, if mongo is selected we don't care about all possible instances for provenance_postgresqldb aeviso: Not sure how to improve this to avoid repeating the test when a fixture that is not used gets a… | |||||
swh_rpc_client: RemoteProvenanceStorage, | swh_rpc_client: RemoteProvenanceStorage, | ||||
) -> ProvenanceStorageInterface: | ) -> ProvenanceStorageInterface: | ||||
"""Return a working and initialized ProvenanceStorageInterface object""" | """Return a working and initialized ProvenanceStorageInterface object""" | ||||
Not Done Inline ActionsWe should probably remove the test database before the tests aeviso: We should probably remove the `test` database before the tests | |||||
if request.param == "remote": | if request.param == "remote": | ||||
assert isinstance(swh_rpc_client, ProvenanceStorageInterface) | assert isinstance(swh_rpc_client, ProvenanceStorageInterface) | ||||
return swh_rpc_client | return swh_rpc_client | ||||
Not Done Inline ActionsThe other storage classes should be enabled again aeviso: The other storage classes should be enabled again | |||||
Done Inline Actions@aeviso I believe we should test all the backends all the time. I will investigate this. jayeshv: @aeviso I believe we should test all the backends all the time. I will investigate this. | |||||
Not Done Inline ActionsI rather keep the previous configuration with a cleaner code. Switching mongo_engine to mongomock in pytest.ini should be enough aeviso: I rather keep the previous configuration with a cleaner code. Switching `mongo_engine` to… | |||||
Not Done Inline ActionsRegarding testing all backends, I agree. That's what I was saying in my comment. Please enable them again aeviso: Regarding testing all backends, I agree. That's what I was saying in my comment. Please enable… | |||||
elif request.param == "mongodb": | |||||
with patch("pymongo.MongoClient.get_database") as patched_mongo: | |||||
patched_mongo.return_value = mongomock.MongoClient().get_database("test") | |||||
return get_provenance_storage(cls=request.param, db={"dbname": "test"}) | |||||
else: | else: | ||||
# in test sessions, we DO want to raise any exception occurring at commit time | # in test sessions, we DO want to raise any exception occurring at commit time | ||||
return get_provenance_storage( | return get_provenance_storage( | ||||
cls=request.param, db=populated_db, raise_on_commit=True | cls=request.param, db=provenance_postgresqldb, raise_on_commit=True | ||||
) | ) | ||||
provenance_postgresql = postgresql("postgresql_proc", dbname="provenance_tests") | provenance_postgresql = postgresql("postgresql_proc", dbname="provenance_tests") | ||||
@pytest.fixture | @pytest.fixture | ||||
def provenance( | def provenance( | ||||
▲ Show 20 Lines • Show All 57 Lines • Show Last 20 Lines |
I believe this include doesn't belong in this group but to the previous one. Pre-commit hooks would fail with it otherwise