Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F7123514
D8695.id31503.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
20 KB
Subscribers
None
D8695.id31503.diff
View Options
diff --git a/swh/provenance/tests/conftest.py b/swh/provenance/tests/conftest.py
--- a/swh/provenance/tests/conftest.py
+++ b/swh/provenance/tests/conftest.py
@@ -3,25 +3,15 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-from contextlib import contextmanager
-from datetime import datetime
from functools import partial
-import multiprocessing
-from os import path
-from pathlib import Path
-from typing import Any, Dict, Generator, List
+from typing import Dict, Generator
from _pytest.fixtures import SubRequest
-from aiohttp.test_utils import TestClient, TestServer, loop_context
-import msgpack
import psycopg2.extensions
import pytest
from pytest_postgresql import factories
from swh.core.db.db_utils import initialize_database_for_module
-from swh.graph.http_rpc_server import make_app
-from swh.journal.serializers import msgpack_ext_hook
-from swh.model.model import BaseModel, TimestampWithTimezone
from swh.provenance import get_provenance
from swh.provenance.archive.interface import ArchiveInterface
from swh.provenance.archive.storage import ArchiveStorage
@@ -30,7 +20,6 @@
from swh.provenance.storage.interface import ProvenanceStorageInterface
from swh.provenance.storage.postgresql import ProvenanceStoragePostgreSql
from swh.storage.interface import StorageInterface
-from swh.storage.replay import OBJECT_CONVERTERS, OBJECT_FIXERS, process_replay_objects
provenance_postgresql_proc = factories.postgresql_proc(
load=[
@@ -93,87 +82,3 @@
def archive(swh_storage: StorageInterface) -> ArchiveInterface:
"""Return an ArchiveStorage-based ArchiveInterface object"""
return ArchiveStorage(swh_storage)
-
-
-def fill_storage(storage: StorageInterface, data: Dict[str, List[dict]]) -> None:
- objects = {
- objtype: [objs_from_dict(objtype, d) for d in dicts]
- for objtype, dicts in data.items()
- }
- process_replay_objects(objects, storage=storage)
-
-
-def get_datafile(fname: str) -> str:
- return path.join(path.dirname(__file__), "data", fname)
-
-
-# TODO: this should return Dict[str, List[BaseModel]] directly, but it requires
-# refactoring several tests
-def load_repo_data(repo: str) -> Dict[str, List[dict]]:
- data: Dict[str, List[dict]] = {}
- with open(get_datafile(f"{repo}.msgpack"), "rb") as fobj:
- unpacker = msgpack.Unpacker(
- fobj,
- raw=False,
- ext_hook=msgpack_ext_hook,
- strict_map_key=False,
- timestamp=3, # convert Timestamp in datetime objects (tz UTC)
- )
- for msg in unpacker:
- if len(msg) == 2: # old format
- objtype, objd = msg
- else: # now we should have a triplet (type, key, value)
- objtype, _, objd = msg
- data.setdefault(objtype, []).append(objd)
- return data
-
-
-def objs_from_dict(object_type: str, dict_repr: dict) -> BaseModel:
- if object_type in OBJECT_FIXERS:
- dict_repr = OBJECT_FIXERS[object_type](dict_repr)
- obj = OBJECT_CONVERTERS[object_type](dict_repr)
- return obj
-
-
-def ts2dt(ts: Dict[str, Any]) -> datetime:
- return TimestampWithTimezone.from_dict(ts).to_datetime()
-
-
-def run_grpc_server(queue, dataset_path):
- try:
- config = {
- "graph": {
- "cls": "local",
- "grpc_server": {"path": dataset_path},
- "http_rpc_server": {"debug": True},
- }
- }
- with loop_context() as loop:
- app = make_app(config=config)
- client = TestClient(TestServer(app), loop=loop)
- loop.run_until_complete(client.start_server())
- url = client.make_url("/graph/")
- queue.put((url, app["rpc_url"]))
- loop.run_forever()
- except Exception as e:
- queue.put(e)
-
-
-@contextmanager
-def grpc_server(dataset):
- dataset_path = (
- Path(__file__).parents[0] / "data/swhgraph" / dataset / "compressed/example"
- )
- queue = multiprocessing.Queue()
- server = multiprocessing.Process(
- target=run_grpc_server, kwargs={"queue": queue, "dataset_path": dataset_path}
- )
- server.start()
- res = queue.get()
- if isinstance(res, Exception):
- raise res
- grpc_url = res[1]
- try:
- yield grpc_url
- finally:
- server.terminate()
diff --git a/swh/provenance/tests/data/generate_graph_dataset.py b/swh/provenance/tests/data/generate_graph_dataset.py
--- a/swh/provenance/tests/data/generate_graph_dataset.py
+++ b/swh/provenance/tests/data/generate_graph_dataset.py
@@ -14,7 +14,7 @@
from swh.dataset.exporters.edges import GraphEdgesExporter
from swh.dataset.exporters.orc import ORCExporter
from swh.graph.webgraph import compress
-from swh.provenance.tests.conftest import load_repo_data
+from swh.provenance.tests.utils import load_repo_data
def main():
diff --git a/swh/provenance/tests/test_archive_interface.py b/swh/provenance/tests/test_archive_interface.py
--- a/swh/provenance/tests/test_archive_interface.py
+++ b/swh/provenance/tests/test_archive_interface.py
@@ -34,10 +34,11 @@
from swh.provenance.archive.postgresql import ArchivePostgreSQL
from swh.provenance.archive.storage import ArchiveStorage
from swh.provenance.archive.swhgraph import ArchiveGraph
-from swh.provenance.tests.conftest import fill_storage, grpc_server, load_repo_data
from swh.storage.interface import StorageInterface
from swh.storage.postgresql.storage import Storage
+from .utils import fill_storage, grpc_server, load_repo_data
+
class ArchiveNoop:
storage: StorageInterface
diff --git a/swh/provenance/tests/test_cli.py b/swh/provenance/tests/test_cli.py
--- a/swh/provenance/tests/test_cli.py
+++ b/swh/provenance/tests/test_cli.py
@@ -22,11 +22,9 @@
from swh.model.hashutil import MultiHash
import swh.provenance.cli # noqa ; ensure cli is loaded
from swh.provenance.storage.interface import EntityType, RelationType
-from swh.provenance.tests.conftest import fill_storage, load_repo_data
from swh.storage.interface import StorageInterface
-from .conftest import get_datafile
-from .test_utils import invoke
+from .utils import fill_storage, get_datafile, invoke, load_repo_data
logger = logging.getLogger(__name__)
diff --git a/swh/provenance/tests/test_conftest.py b/swh/provenance/tests/test_conftest.py
--- a/swh/provenance/tests/test_conftest.py
+++ b/swh/provenance/tests/test_conftest.py
@@ -4,9 +4,10 @@
# See top-level LICENSE file for more information
from swh.provenance.interface import ProvenanceInterface
-from swh.provenance.tests.conftest import fill_storage, load_repo_data
from swh.storage.interface import StorageInterface
+from .utils import fill_storage, load_repo_data
+
def test_provenance_fixture(provenance: ProvenanceInterface) -> None:
"""Check the 'provenance' fixture produce a working
diff --git a/swh/provenance/tests/test_consistency.py b/swh/provenance/tests/test_consistency.py
--- a/swh/provenance/tests/test_consistency.py
+++ b/swh/provenance/tests/test_consistency.py
@@ -9,7 +9,8 @@
from swh.provenance.interface import ProvenanceInterface
from swh.provenance.model import RevisionEntry
from swh.provenance.storage.interface import DirectoryData, ProvenanceResult
-from swh.provenance.tests.conftest import fill_storage, load_repo_data, ts2dt
+
+from .utils import fill_storage, load_repo_data, ts2dt
def test_consistency(
diff --git a/swh/provenance/tests/test_directory_flatten.py b/swh/provenance/tests/test_directory_flatten.py
--- a/swh/provenance/tests/test_directory_flatten.py
+++ b/swh/provenance/tests/test_directory_flatten.py
@@ -13,7 +13,8 @@
from swh.provenance.interface import ProvenanceInterface
from swh.provenance.model import DirectoryEntry, FileEntry
from swh.provenance.storage.interface import DirectoryData, RelationData, RelationType
-from swh.provenance.tests.conftest import fill_storage, load_repo_data
+
+from .utils import fill_storage, load_repo_data
def prepare(
diff --git a/swh/provenance/tests/test_directory_iterator.py b/swh/provenance/tests/test_directory_iterator.py
--- a/swh/provenance/tests/test_directory_iterator.py
+++ b/swh/provenance/tests/test_directory_iterator.py
@@ -6,9 +6,10 @@
import pytest
from swh.provenance.algos.directory import CSVDirectoryIterator
-from swh.provenance.tests.conftest import fill_storage, load_repo_data
from swh.storage.interface import StorageInterface
+from .utils import fill_storage, load_repo_data
+
@pytest.mark.parametrize(
"repo",
diff --git a/swh/provenance/tests/test_history_graph.py b/swh/provenance/tests/test_history_graph.py
--- a/swh/provenance/tests/test_history_graph.py
+++ b/swh/provenance/tests/test_history_graph.py
@@ -11,7 +11,8 @@
from swh.provenance.archive import ArchiveInterface
from swh.provenance.interface import ProvenanceInterface
from swh.provenance.model import OriginEntry, RevisionEntry
-from swh.provenance.tests.conftest import fill_storage, get_datafile, load_repo_data
+
+from .utils import fill_storage, get_datafile, load_repo_data
@pytest.mark.origin_layer
diff --git a/swh/provenance/tests/test_isochrone_graph.py b/swh/provenance/tests/test_isochrone_graph.py
--- a/swh/provenance/tests/test_isochrone_graph.py
+++ b/swh/provenance/tests/test_isochrone_graph.py
@@ -20,12 +20,8 @@
from swh.provenance.archive import ArchiveInterface
from swh.provenance.interface import ProvenanceInterface
from swh.provenance.model import DirectoryEntry, RevisionEntry
-from swh.provenance.tests.conftest import (
- fill_storage,
- get_datafile,
- load_repo_data,
- ts2dt,
-)
+
+from .utils import fill_storage, get_datafile, load_repo_data, ts2dt
def isochrone_graph_from_dict(d: Dict[str, Any], depth: int = 0) -> IsochroneNode:
diff --git a/swh/provenance/tests/test_journal_client.py b/swh/provenance/tests/test_journal_client.py
--- a/swh/provenance/tests/test_journal_client.py
+++ b/swh/provenance/tests/test_journal_client.py
@@ -9,10 +9,9 @@
import pytest
from swh.model.hashutil import MultiHash
-from swh.provenance.tests.conftest import fill_storage, load_repo_data
from swh.storage.interface import StorageInterface
-from .test_utils import invoke
+from .utils import fill_storage, invoke, load_repo_data
@pytest.fixture
@@ -70,8 +69,8 @@
}
# call the cli 'swh provenance origin from-journal'
- result = invoke(["origin", "from-journal"], config=cfg)
- assert result.exit_code == 0, f"Unexpected result: {result.output}"
+ cli_result = invoke(["origin", "from-journal"], config=cfg)
+ assert cli_result.exit_code == 0, f"Unexpected result: {cli_result.output}"
origin_sha1 = MultiHash.from_data(
origin_url.encode(), hash_names=["sha1"]
@@ -127,7 +126,7 @@
# call the cli 'swh provenance revision from-journal'
cli_result = invoke(["revision", "from-journal"], config=cfg)
- assert cli_result.exit_code == 0, f"Unexpected result: {result.output}"
+ assert cli_result.exit_code == 0, f"Unexpected result: {cli_result.output}"
result = provenance.storage.revision_get(revisions)
diff --git a/swh/provenance/tests/test_origin_iterator.py b/swh/provenance/tests/test_origin_iterator.py
--- a/swh/provenance/tests/test_origin_iterator.py
+++ b/swh/provenance/tests/test_origin_iterator.py
@@ -6,7 +6,6 @@
import pytest
from swh.provenance.algos.origin import CSVOriginIterator
-from swh.provenance.tests.conftest import fill_storage, load_repo_data
from swh.storage.algos.origin import (
iter_origin_visit_statuses,
iter_origin_visits,
@@ -14,6 +13,8 @@
)
from swh.storage.interface import StorageInterface
+from .utils import fill_storage, load_repo_data
+
@pytest.mark.origin_layer
@pytest.mark.parametrize(
diff --git a/swh/provenance/tests/test_origin_revision_layer.py b/swh/provenance/tests/test_origin_revision_layer.py
--- a/swh/provenance/tests/test_origin_revision_layer.py
+++ b/swh/provenance/tests/test_origin_revision_layer.py
@@ -16,7 +16,8 @@
from swh.provenance.interface import ProvenanceInterface
from swh.provenance.model import OriginEntry
from swh.provenance.storage.interface import EntityType, RelationType
-from swh.provenance.tests.conftest import fill_storage, get_datafile, load_repo_data
+
+from .utils import fill_storage, get_datafile, load_repo_data
class SynthRelation(TypedDict):
diff --git a/swh/provenance/tests/test_provenance_storage.py b/swh/provenance/tests/test_provenance_storage.py
--- a/swh/provenance/tests/test_provenance_storage.py
+++ b/swh/provenance/tests/test_provenance_storage.py
@@ -28,7 +28,8 @@
RelationType,
RevisionData,
)
-from swh.provenance.tests.conftest import fill_storage, load_repo_data, ts2dt
+
+from .utils import fill_storage, load_repo_data, ts2dt
class TestProvenanceStorage:
diff --git a/swh/provenance/tests/test_replay.py b/swh/provenance/tests/test_replay.py
--- a/swh/provenance/tests/test_replay.py
+++ b/swh/provenance/tests/test_replay.py
@@ -25,7 +25,7 @@
process_replay_objects,
)
-from .conftest import fill_storage, load_repo_data, ts2dt
+from .utils import fill_storage, load_repo_data, ts2dt
@pytest.fixture(scope="function")
diff --git a/swh/provenance/tests/test_revision_content_layer.py b/swh/provenance/tests/test_revision_content_layer.py
--- a/swh/provenance/tests/test_revision_content_layer.py
+++ b/swh/provenance/tests/test_revision_content_layer.py
@@ -17,12 +17,8 @@
from swh.provenance.interface import ProvenanceInterface
from swh.provenance.model import DirectoryEntry, RevisionEntry
from swh.provenance.storage.interface import EntityType, RelationType
-from swh.provenance.tests.conftest import (
- fill_storage,
- get_datafile,
- load_repo_data,
- ts2dt,
-)
+
+from .utils import fill_storage, get_datafile, load_repo_data, ts2dt
class SynthRelation(TypedDict):
diff --git a/swh/provenance/tests/test_revision_iterator.py b/swh/provenance/tests/test_revision_iterator.py
--- a/swh/provenance/tests/test_revision_iterator.py
+++ b/swh/provenance/tests/test_revision_iterator.py
@@ -6,9 +6,10 @@
import pytest
from swh.provenance.algos.revision import CSVRevisionIterator
-from swh.provenance.tests.conftest import fill_storage, load_repo_data, ts2dt
from swh.storage.interface import StorageInterface
+from .utils import fill_storage, load_repo_data, ts2dt
+
@pytest.mark.parametrize(
"repo",
diff --git a/swh/provenance/tests/test_utils.py b/swh/provenance/tests/test_utils.py
deleted file mode 100644
--- a/swh/provenance/tests/test_utils.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# Copyright (C) 2022 The Software Heritage developers
-# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
-# See top-level LICENSE file for more information
-
-
-import logging
-import tempfile
-from typing import Dict, List, Optional
-
-from click.testing import CliRunner, Result
-from yaml import safe_dump
-
-from swh.provenance.cli import cli
-
-
-def invoke(
- args: List[str], config: Optional[Dict] = None, catch_exceptions: bool = False
-) -> Result:
- """Invoke swh journal subcommands"""
- runner = CliRunner()
- with tempfile.NamedTemporaryFile("a", suffix=".yml") as config_fd:
- if config is not None:
- safe_dump(config, config_fd)
- config_fd.seek(0)
- args = ["-C" + config_fd.name] + args
-
- result = runner.invoke(cli, args, obj={"log_level": logging.DEBUG}, env=None)
- if not catch_exceptions and result.exception:
- print(result.output)
- raise result.exception
- return result
diff --git a/swh/provenance/tests/conftest.py b/swh/provenance/tests/utils.py
copy from swh/provenance/tests/conftest.py
copy to swh/provenance/tests/utils.py
--- a/swh/provenance/tests/conftest.py
+++ b/swh/provenance/tests/utils.py
@@ -1,98 +1,47 @@
-# Copyright (C) 2021-2022 The Software Heritage developers
+# Copyright (C) 2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
+
from contextlib import contextmanager
from datetime import datetime
-from functools import partial
+import logging
import multiprocessing
from os import path
from pathlib import Path
-from typing import Any, Dict, Generator, List
+import tempfile
+from typing import Any, Dict, List, Optional
-from _pytest.fixtures import SubRequest
from aiohttp.test_utils import TestClient, TestServer, loop_context
+from click.testing import CliRunner, Result
import msgpack
-import psycopg2.extensions
-import pytest
-from pytest_postgresql import factories
+from yaml import safe_dump
-from swh.core.db.db_utils import initialize_database_for_module
from swh.graph.http_rpc_server import make_app
from swh.journal.serializers import msgpack_ext_hook
from swh.model.model import BaseModel, TimestampWithTimezone
-from swh.provenance import get_provenance
-from swh.provenance.archive.interface import ArchiveInterface
-from swh.provenance.archive.storage import ArchiveStorage
-from swh.provenance.interface import ProvenanceInterface
-from swh.provenance.storage import get_provenance_storage
-from swh.provenance.storage.interface import ProvenanceStorageInterface
-from swh.provenance.storage.postgresql import ProvenanceStoragePostgreSql
+from swh.provenance.cli import cli
from swh.storage.interface import StorageInterface
from swh.storage.replay import OBJECT_CONVERTERS, OBJECT_FIXERS, process_replay_objects
-provenance_postgresql_proc = factories.postgresql_proc(
- load=[
- partial(
- initialize_database_for_module,
- modname="provenance",
- flavor="normalized",
- version=ProvenanceStoragePostgreSql.current_version,
- )
- ],
-)
-
-postgres_provenance = factories.postgresql("provenance_postgresql_proc")
-
-
-@pytest.fixture()
-def provenance_postgresqldb(request, postgres_provenance):
- return postgres_provenance.get_dsn_parameters()
-
-
-@pytest.fixture()
-def provenance_storage(
- request: SubRequest,
- provenance_postgresqldb: Dict[str, str],
-) -> Generator[ProvenanceStorageInterface, None, None]:
- """Return a working and initialized ProvenanceStorageInterface object"""
- # in test sessions, we DO want to raise any exception occurring at commit time
- with get_provenance_storage(
- cls="postgresql", db=provenance_postgresqldb, raise_on_commit=True
- ) as storage:
- yield storage
-
-
-@pytest.fixture
-def provenance(
- postgres_provenance: psycopg2.extensions.connection,
-) -> Generator[ProvenanceInterface, None, None]:
- """Return a working and initialized ProvenanceInterface object"""
-
- from swh.core.db.db_utils import (
- init_admin_extensions,
- populate_database_for_package,
- )
-
- init_admin_extensions("swh.provenance", postgres_provenance.dsn)
- populate_database_for_package(
- "swh.provenance", postgres_provenance.dsn, flavor="normalized"
- )
- # in test sessions, we DO want to raise any exception occurring at commit time
- with get_provenance(
- cls="postgresql",
- db=postgres_provenance.get_dsn_parameters(),
- raise_on_commit=True,
- ) as provenance:
- yield provenance
-
-
-@pytest.fixture
-def archive(swh_storage: StorageInterface) -> ArchiveInterface:
- """Return an ArchiveStorage-based ArchiveInterface object"""
- return ArchiveStorage(swh_storage)
+def invoke(
+ args: List[str], config: Optional[Dict] = None, catch_exceptions: bool = False
+) -> Result:
+ """Invoke swh journal subcommands"""
+ runner = CliRunner()
+ with tempfile.NamedTemporaryFile("a", suffix=".yml") as config_fd:
+ if config is not None:
+ safe_dump(config, config_fd)
+ config_fd.seek(0)
+ args = ["-C" + config_fd.name] + args
+
+ result = runner.invoke(cli, args, obj={"log_level": logging.DEBUG}, env=None)
+ if not catch_exceptions and result.exception:
+ print(result.output)
+ raise result.exception
+ return result
def fill_storage(storage: StorageInterface, data: Dict[str, List[dict]]) -> None:
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Thu, Dec 19, 9:58 AM (15 h, 42 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3231759
Attached To
D8695: Move all non pytest-related functions from conftest to utils.py
Event Timeline
Log In to Comment