diff --git a/swh/provenance/tests/conftest.py b/swh/provenance/tests/conftest.py --- a/swh/provenance/tests/conftest.py +++ b/swh/provenance/tests/conftest.py @@ -3,25 +3,15 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from contextlib import contextmanager -from datetime import datetime from functools import partial -import multiprocessing -from os import path -from pathlib import Path -from typing import Any, Dict, Generator, List +from typing import Dict, Generator from _pytest.fixtures import SubRequest -from aiohttp.test_utils import TestClient, TestServer, loop_context -import msgpack import psycopg2.extensions import pytest from pytest_postgresql import factories from swh.core.db.db_utils import initialize_database_for_module -from swh.graph.http_rpc_server import make_app -from swh.journal.serializers import msgpack_ext_hook -from swh.model.model import BaseModel, TimestampWithTimezone from swh.provenance import get_provenance from swh.provenance.archive.interface import ArchiveInterface from swh.provenance.archive.storage import ArchiveStorage @@ -30,7 +20,6 @@ from swh.provenance.storage.interface import ProvenanceStorageInterface from swh.provenance.storage.postgresql import ProvenanceStoragePostgreSql from swh.storage.interface import StorageInterface -from swh.storage.replay import OBJECT_CONVERTERS, OBJECT_FIXERS, process_replay_objects provenance_postgresql_proc = factories.postgresql_proc( load=[ @@ -93,87 +82,3 @@ def archive(swh_storage: StorageInterface) -> ArchiveInterface: """Return an ArchiveStorage-based ArchiveInterface object""" return ArchiveStorage(swh_storage) - - -def fill_storage(storage: StorageInterface, data: Dict[str, List[dict]]) -> None: - objects = { - objtype: [objs_from_dict(objtype, d) for d in dicts] - for objtype, dicts in data.items() - } - process_replay_objects(objects, storage=storage) - - -def get_datafile(fname: str) -> str: - return path.join(path.dirname(__file__), "data", fname) - - -# TODO: this should return Dict[str, List[BaseModel]] directly, but it requires -# refactoring several tests -def load_repo_data(repo: str) -> Dict[str, List[dict]]: - data: Dict[str, List[dict]] = {} - with open(get_datafile(f"{repo}.msgpack"), "rb") as fobj: - unpacker = msgpack.Unpacker( - fobj, - raw=False, - ext_hook=msgpack_ext_hook, - strict_map_key=False, - timestamp=3, # convert Timestamp in datetime objects (tz UTC) - ) - for msg in unpacker: - if len(msg) == 2: # old format - objtype, objd = msg - else: # now we should have a triplet (type, key, value) - objtype, _, objd = msg - data.setdefault(objtype, []).append(objd) - return data - - -def objs_from_dict(object_type: str, dict_repr: dict) -> BaseModel: - if object_type in OBJECT_FIXERS: - dict_repr = OBJECT_FIXERS[object_type](dict_repr) - obj = OBJECT_CONVERTERS[object_type](dict_repr) - return obj - - -def ts2dt(ts: Dict[str, Any]) -> datetime: - return TimestampWithTimezone.from_dict(ts).to_datetime() - - -def run_grpc_server(queue, dataset_path): - try: - config = { - "graph": { - "cls": "local", - "grpc_server": {"path": dataset_path}, - "http_rpc_server": {"debug": True}, - } - } - with loop_context() as loop: - app = make_app(config=config) - client = TestClient(TestServer(app), loop=loop) - loop.run_until_complete(client.start_server()) - url = client.make_url("/graph/") - queue.put((url, app["rpc_url"])) - loop.run_forever() - except Exception as e: - queue.put(e) - - -@contextmanager -def grpc_server(dataset): - dataset_path = ( - Path(__file__).parents[0] / "data/swhgraph" / dataset / "compressed/example" - ) - queue = multiprocessing.Queue() - server = multiprocessing.Process( - target=run_grpc_server, kwargs={"queue": queue, "dataset_path": dataset_path} - ) - server.start() - res = queue.get() - if isinstance(res, Exception): - raise res - grpc_url = res[1] - try: - yield grpc_url - finally: - server.terminate() diff --git a/swh/provenance/tests/data/generate_graph_dataset.py b/swh/provenance/tests/data/generate_graph_dataset.py --- a/swh/provenance/tests/data/generate_graph_dataset.py +++ b/swh/provenance/tests/data/generate_graph_dataset.py @@ -14,7 +14,7 @@ from swh.dataset.exporters.edges import GraphEdgesExporter from swh.dataset.exporters.orc import ORCExporter from swh.graph.webgraph import compress -from swh.provenance.tests.conftest import load_repo_data +from swh.provenance.tests.utils import load_repo_data def main(): diff --git a/swh/provenance/tests/test_archive_interface.py b/swh/provenance/tests/test_archive_interface.py --- a/swh/provenance/tests/test_archive_interface.py +++ b/swh/provenance/tests/test_archive_interface.py @@ -34,10 +34,11 @@ from swh.provenance.archive.postgresql import ArchivePostgreSQL from swh.provenance.archive.storage import ArchiveStorage from swh.provenance.archive.swhgraph import ArchiveGraph -from swh.provenance.tests.conftest import fill_storage, grpc_server, load_repo_data from swh.storage.interface import StorageInterface from swh.storage.postgresql.storage import Storage +from .utils import fill_storage, grpc_server, load_repo_data + class ArchiveNoop: storage: StorageInterface diff --git a/swh/provenance/tests/test_cli.py b/swh/provenance/tests/test_cli.py --- a/swh/provenance/tests/test_cli.py +++ b/swh/provenance/tests/test_cli.py @@ -22,11 +22,9 @@ from swh.model.hashutil import MultiHash import swh.provenance.cli # noqa ; ensure cli is loaded from swh.provenance.storage.interface import EntityType, RelationType -from swh.provenance.tests.conftest import fill_storage, load_repo_data from swh.storage.interface import StorageInterface -from .conftest import get_datafile -from .test_utils import invoke +from .utils import fill_storage, get_datafile, invoke, load_repo_data logger = logging.getLogger(__name__) diff --git a/swh/provenance/tests/test_conftest.py b/swh/provenance/tests/test_conftest.py --- a/swh/provenance/tests/test_conftest.py +++ b/swh/provenance/tests/test_conftest.py @@ -4,9 +4,10 @@ # See top-level LICENSE file for more information from swh.provenance.interface import ProvenanceInterface -from swh.provenance.tests.conftest import fill_storage, load_repo_data from swh.storage.interface import StorageInterface +from .utils import fill_storage, load_repo_data + def test_provenance_fixture(provenance: ProvenanceInterface) -> None: """Check the 'provenance' fixture produce a working diff --git a/swh/provenance/tests/test_consistency.py b/swh/provenance/tests/test_consistency.py --- a/swh/provenance/tests/test_consistency.py +++ b/swh/provenance/tests/test_consistency.py @@ -9,7 +9,8 @@ from swh.provenance.interface import ProvenanceInterface from swh.provenance.model import RevisionEntry from swh.provenance.storage.interface import DirectoryData, ProvenanceResult -from swh.provenance.tests.conftest import fill_storage, load_repo_data, ts2dt + +from .utils import fill_storage, load_repo_data, ts2dt def test_consistency( diff --git a/swh/provenance/tests/test_directory_flatten.py b/swh/provenance/tests/test_directory_flatten.py --- a/swh/provenance/tests/test_directory_flatten.py +++ b/swh/provenance/tests/test_directory_flatten.py @@ -13,7 +13,8 @@ from swh.provenance.interface import ProvenanceInterface from swh.provenance.model import DirectoryEntry, FileEntry from swh.provenance.storage.interface import DirectoryData, RelationData, RelationType -from swh.provenance.tests.conftest import fill_storage, load_repo_data + +from .utils import fill_storage, load_repo_data def prepare( diff --git a/swh/provenance/tests/test_directory_iterator.py b/swh/provenance/tests/test_directory_iterator.py --- a/swh/provenance/tests/test_directory_iterator.py +++ b/swh/provenance/tests/test_directory_iterator.py @@ -6,9 +6,10 @@ import pytest from swh.provenance.algos.directory import CSVDirectoryIterator -from swh.provenance.tests.conftest import fill_storage, load_repo_data from swh.storage.interface import StorageInterface +from .utils import fill_storage, load_repo_data + @pytest.mark.parametrize( "repo", diff --git a/swh/provenance/tests/test_history_graph.py b/swh/provenance/tests/test_history_graph.py --- a/swh/provenance/tests/test_history_graph.py +++ b/swh/provenance/tests/test_history_graph.py @@ -11,7 +11,8 @@ from swh.provenance.archive import ArchiveInterface from swh.provenance.interface import ProvenanceInterface from swh.provenance.model import OriginEntry, RevisionEntry -from swh.provenance.tests.conftest import fill_storage, get_datafile, load_repo_data + +from .utils import fill_storage, get_datafile, load_repo_data @pytest.mark.origin_layer diff --git a/swh/provenance/tests/test_isochrone_graph.py b/swh/provenance/tests/test_isochrone_graph.py --- a/swh/provenance/tests/test_isochrone_graph.py +++ b/swh/provenance/tests/test_isochrone_graph.py @@ -20,12 +20,8 @@ from swh.provenance.archive import ArchiveInterface from swh.provenance.interface import ProvenanceInterface from swh.provenance.model import DirectoryEntry, RevisionEntry -from swh.provenance.tests.conftest import ( - fill_storage, - get_datafile, - load_repo_data, - ts2dt, -) + +from .utils import fill_storage, get_datafile, load_repo_data, ts2dt def isochrone_graph_from_dict(d: Dict[str, Any], depth: int = 0) -> IsochroneNode: diff --git a/swh/provenance/tests/test_journal_client.py b/swh/provenance/tests/test_journal_client.py --- a/swh/provenance/tests/test_journal_client.py +++ b/swh/provenance/tests/test_journal_client.py @@ -9,10 +9,9 @@ import pytest from swh.model.hashutil import MultiHash -from swh.provenance.tests.conftest import fill_storage, load_repo_data from swh.storage.interface import StorageInterface -from .test_utils import invoke +from .utils import fill_storage, invoke, load_repo_data @pytest.fixture diff --git a/swh/provenance/tests/test_origin_iterator.py b/swh/provenance/tests/test_origin_iterator.py --- a/swh/provenance/tests/test_origin_iterator.py +++ b/swh/provenance/tests/test_origin_iterator.py @@ -6,7 +6,6 @@ import pytest from swh.provenance.algos.origin import CSVOriginIterator -from swh.provenance.tests.conftest import fill_storage, load_repo_data from swh.storage.algos.origin import ( iter_origin_visit_statuses, iter_origin_visits, @@ -14,6 +13,8 @@ ) from swh.storage.interface import StorageInterface +from .utils import fill_storage, load_repo_data + @pytest.mark.origin_layer @pytest.mark.parametrize( diff --git a/swh/provenance/tests/test_origin_revision_layer.py b/swh/provenance/tests/test_origin_revision_layer.py --- a/swh/provenance/tests/test_origin_revision_layer.py +++ b/swh/provenance/tests/test_origin_revision_layer.py @@ -16,7 +16,8 @@ from swh.provenance.interface import ProvenanceInterface from swh.provenance.model import OriginEntry from swh.provenance.storage.interface import EntityType, RelationType -from swh.provenance.tests.conftest import fill_storage, get_datafile, load_repo_data + +from .utils import fill_storage, get_datafile, load_repo_data class SynthRelation(TypedDict): diff --git a/swh/provenance/tests/test_provenance_storage.py b/swh/provenance/tests/test_provenance_storage.py --- a/swh/provenance/tests/test_provenance_storage.py +++ b/swh/provenance/tests/test_provenance_storage.py @@ -28,7 +28,8 @@ RelationType, RevisionData, ) -from swh.provenance.tests.conftest import fill_storage, load_repo_data, ts2dt + +from .utils import fill_storage, load_repo_data, ts2dt class TestProvenanceStorage: diff --git a/swh/provenance/tests/test_replay.py b/swh/provenance/tests/test_replay.py --- a/swh/provenance/tests/test_replay.py +++ b/swh/provenance/tests/test_replay.py @@ -25,7 +25,7 @@ process_replay_objects, ) -from .conftest import fill_storage, load_repo_data, ts2dt +from .utils import fill_storage, load_repo_data, ts2dt @pytest.fixture(scope="function") diff --git a/swh/provenance/tests/test_revision_content_layer.py b/swh/provenance/tests/test_revision_content_layer.py --- a/swh/provenance/tests/test_revision_content_layer.py +++ b/swh/provenance/tests/test_revision_content_layer.py @@ -17,12 +17,8 @@ from swh.provenance.interface import ProvenanceInterface from swh.provenance.model import DirectoryEntry, RevisionEntry from swh.provenance.storage.interface import EntityType, RelationType -from swh.provenance.tests.conftest import ( - fill_storage, - get_datafile, - load_repo_data, - ts2dt, -) + +from .utils import fill_storage, get_datafile, load_repo_data, ts2dt class SynthRelation(TypedDict): diff --git a/swh/provenance/tests/test_revision_iterator.py b/swh/provenance/tests/test_revision_iterator.py --- a/swh/provenance/tests/test_revision_iterator.py +++ b/swh/provenance/tests/test_revision_iterator.py @@ -6,9 +6,10 @@ import pytest from swh.provenance.algos.revision import CSVRevisionIterator -from swh.provenance.tests.conftest import fill_storage, load_repo_data, ts2dt from swh.storage.interface import StorageInterface +from .utils import fill_storage, load_repo_data, ts2dt + @pytest.mark.parametrize( "repo", diff --git a/swh/provenance/tests/test_utils.py b/swh/provenance/tests/test_utils.py deleted file mode 100644 --- a/swh/provenance/tests/test_utils.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (C) 2022 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - - -import logging -import tempfile -from typing import Dict, List, Optional - -from click.testing import CliRunner, Result -from yaml import safe_dump - -from swh.provenance.cli import cli - - -def invoke( - args: List[str], config: Optional[Dict] = None, catch_exceptions: bool = False -) -> Result: - """Invoke swh journal subcommands""" - runner = CliRunner() - with tempfile.NamedTemporaryFile("a", suffix=".yml") as config_fd: - if config is not None: - safe_dump(config, config_fd) - config_fd.seek(0) - args = ["-C" + config_fd.name] + args - - result = runner.invoke(cli, args, obj={"log_level": logging.DEBUG}, env=None) - if not catch_exceptions and result.exception: - print(result.output) - raise result.exception - return result diff --git a/swh/provenance/tests/conftest.py b/swh/provenance/tests/utils.py copy from swh/provenance/tests/conftest.py copy to swh/provenance/tests/utils.py --- a/swh/provenance/tests/conftest.py +++ b/swh/provenance/tests/utils.py @@ -1,98 +1,47 @@ -# Copyright (C) 2021-2022 The Software Heritage developers +# Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information + from contextlib import contextmanager from datetime import datetime -from functools import partial +import logging import multiprocessing from os import path from pathlib import Path -from typing import Any, Dict, Generator, List +import tempfile +from typing import Any, Dict, List, Optional -from _pytest.fixtures import SubRequest from aiohttp.test_utils import TestClient, TestServer, loop_context +from click.testing import CliRunner, Result import msgpack -import psycopg2.extensions -import pytest -from pytest_postgresql import factories +from yaml import safe_dump -from swh.core.db.db_utils import initialize_database_for_module from swh.graph.http_rpc_server import make_app from swh.journal.serializers import msgpack_ext_hook from swh.model.model import BaseModel, TimestampWithTimezone -from swh.provenance import get_provenance -from swh.provenance.archive.interface import ArchiveInterface -from swh.provenance.archive.storage import ArchiveStorage -from swh.provenance.interface import ProvenanceInterface -from swh.provenance.storage import get_provenance_storage -from swh.provenance.storage.interface import ProvenanceStorageInterface -from swh.provenance.storage.postgresql import ProvenanceStoragePostgreSql +from swh.provenance.cli import cli from swh.storage.interface import StorageInterface from swh.storage.replay import OBJECT_CONVERTERS, OBJECT_FIXERS, process_replay_objects -provenance_postgresql_proc = factories.postgresql_proc( - load=[ - partial( - initialize_database_for_module, - modname="provenance", - flavor="normalized", - version=ProvenanceStoragePostgreSql.current_version, - ) - ], -) - -postgres_provenance = factories.postgresql("provenance_postgresql_proc") - - -@pytest.fixture() -def provenance_postgresqldb(request, postgres_provenance): - return postgres_provenance.get_dsn_parameters() - - -@pytest.fixture() -def provenance_storage( - request: SubRequest, - provenance_postgresqldb: Dict[str, str], -) -> Generator[ProvenanceStorageInterface, None, None]: - """Return a working and initialized ProvenanceStorageInterface object""" - # in test sessions, we DO want to raise any exception occurring at commit time - with get_provenance_storage( - cls="postgresql", db=provenance_postgresqldb, raise_on_commit=True - ) as storage: - yield storage - - -@pytest.fixture -def provenance( - postgres_provenance: psycopg2.extensions.connection, -) -> Generator[ProvenanceInterface, None, None]: - """Return a working and initialized ProvenanceInterface object""" - - from swh.core.db.db_utils import ( - init_admin_extensions, - populate_database_for_package, - ) - - init_admin_extensions("swh.provenance", postgres_provenance.dsn) - populate_database_for_package( - "swh.provenance", postgres_provenance.dsn, flavor="normalized" - ) - # in test sessions, we DO want to raise any exception occurring at commit time - with get_provenance( - cls="postgresql", - db=postgres_provenance.get_dsn_parameters(), - raise_on_commit=True, - ) as provenance: - yield provenance - - -@pytest.fixture -def archive(swh_storage: StorageInterface) -> ArchiveInterface: - """Return an ArchiveStorage-based ArchiveInterface object""" - return ArchiveStorage(swh_storage) +def invoke( + args: List[str], config: Optional[Dict] = None, catch_exceptions: bool = False +) -> Result: + """Invoke swh journal subcommands""" + runner = CliRunner() + with tempfile.NamedTemporaryFile("a", suffix=".yml") as config_fd: + if config is not None: + safe_dump(config, config_fd) + config_fd.seek(0) + args = ["-C" + config_fd.name] + args + + result = runner.invoke(cli, args, obj={"log_level": logging.DEBUG}, env=None) + if not catch_exceptions and result.exception: + print(result.output) + raise result.exception + return result def fill_storage(storage: StorageInterface, data: Dict[str, List[dict]]) -> None: