Differential D6094 Diff 22152 swh/provenance/tests/conftest.py

Changeset View

Standalone View

swh/provenance/tests/conftest.py

	# Copyright (C) 2021 The Software Heritage developers			# Copyright (C) 2021 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution			# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version			# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information			# See top-level LICENSE file for more information

	from datetime import datetime, timedelta, timezone			from datetime import datetime, timedelta, timezone
	from os import path			from os import path
	from typing import Any, Dict, Iterable, Iterator			from typing import Any, Dict, Iterable, Iterator

	from _pytest.fixtures import SubRequest			from _pytest.fixtures import SubRequest
	import msgpack			import msgpack
	import psycopg2.extensions			import psycopg2.extensions
				import pymongo.database
	import pytest			import pytest
	from pytest_postgresql.factories import postgresql			from pytest_postgresql.factories import postgresql

	from swh.journal.serializers import msgpack_ext_hook			from swh.journal.serializers import msgpack_ext_hook
	from swh.provenance import get_provenance, get_provenance_storage			from swh.provenance import get_provenance, get_provenance_storage
	from swh.provenance.api.client import RemoteProvenanceStorage			from swh.provenance.api.client import RemoteProvenanceStorage
	import swh.provenance.api.server as server			import swh.provenance.api.server as server
	from swh.provenance.archive import ArchiveInterface			from swh.provenance.archive import ArchiveInterface
	from swh.provenance.interface import ProvenanceInterface, ProvenanceStorageInterface			from swh.provenance.interface import ProvenanceInterface, ProvenanceStorageInterface
	from swh.provenance.storage.archive import ArchiveStorage			from swh.provenance.storage.archive import ArchiveStorage
	from swh.storage.interface import StorageInterface			from swh.storage.interface import StorageInterface
	from swh.storage.replay import process_replay_objects			from swh.storage.replay import process_replay_objects
				aevisoUnsubmitted Not Done Inline Actions I believe this include doesn't belong in this group but to the previous one. Pre-commit hooks would fail with it otherwise aeviso: I believe this include doesn't belong in this group but to the previous one. Pre-commit hooks…


	@pytest.fixture(			@pytest.fixture(
	params=[			params=[
	"with-path",			"with-path",
	"without-path",			"without-path",
	"with-path-denormalized",			"with-path-denormalized",
	"without-path-denormalized",			"without-path-denormalized",
	]			]
	)			)
	def populated_db(			def provenance_postgresqldb(
	request: SubRequest,			request: SubRequest,
	postgresql: psycopg2.extensions.connection,			postgresql: psycopg2.extensions.connection,
	) -> Dict[str, str]:			) -> Dict[str, str]:
	"""return a working and initialized provenance db"""			"""return a working and initialized provenance db"""
	from swh.core.cli.db import populate_database_for_package			from swh.core.cli.db import populate_database_for_package

	populate_database_for_package(			populate_database_for_package(
	"swh.provenance", postgresql.dsn, flavor=request.param			"swh.provenance", postgresql.dsn, flavor=request.param
	)			)
	return postgresql.get_dsn_parameters()			return postgresql.get_dsn_parameters()


	# the Flask app used as server in these tests			# the Flask app used as server in these tests
	@pytest.fixture			@pytest.fixture
	def app(populated_db: Dict[str, str]) -> Iterator[server.ProvenanceStorageServerApp]:			def app(
				provenance_postgresqldb: Dict[str, str]
				) -> Iterator[server.ProvenanceStorageServerApp]:
	assert hasattr(server, "storage")			assert hasattr(server, "storage")
	server.storage = get_provenance_storage(cls="postgresql", db=populated_db)			server.storage = get_provenance_storage(
				cls="postgresql", db=provenance_postgresqldb
				)
	yield server.app			yield server.app


	# the RPCClient class used as client used in these tests			# the RPCClient class used as client used in these tests
	@pytest.fixture			@pytest.fixture
	def swh_rpc_client_class() -> type:			def swh_rpc_client_class() -> type:
	return RemoteProvenanceStorage			return RemoteProvenanceStorage


	@pytest.fixture(params=["postgresql", "remote"])			@pytest.fixture
				def provenance_mongodb(mongodb: pymongo.database.Database) -> Dict[str, str]:
				"""Return an empty test Mongo database"""

				for collection in mongodb.list_collection_names():
				print("Dropping collection", collection)
				mongodb.drop_collection(collection)

				aevisoUnsubmitted Not Done Inline Actions We should probably remove the `test` database before the tests aeviso: We should probably remove the `test` database before the tests
				return {"dbname": mongodb.name}


				aevisoUnsubmitted Not Done Inline Actions The other storage classes should be enabled again aeviso: The other storage classes should be enabled again
				jayeshvAuthorUnsubmitted Done Inline Actions @aeviso I believe we should test all the backends all the time. I will investigate this. Another problem is, we have an in-memory implementation to test the postgres backend. For mongo, it is expecting a real server and the tests will fail. I will fix this withe a mock at this point of time. jayeshv: @aeviso I believe we should test all the backends all the time. I will investigate this.
				aevisoUnsubmitted Not Done Inline Actions I rather keep the previous configuration with a cleaner code. Switching `mongo_engine` to `mongomock` in `pytest.ini` should be enough aeviso: I rather keep the previous configuration with a cleaner code. Switching `mongo_engine` to…
				aevisoUnsubmitted Not Done Inline Actions Regarding testing all backends, I agree. That's what I was saying in my comment. Please enable them again aeviso: Regarding testing all backends, I agree. That's what I was saying in my comment. Please enable…
				@pytest.fixture(params=["mongodb"])
	def provenance_storage(			def provenance_storage(
	request: SubRequest,			request: SubRequest,
	populated_db: Dict[str, str],			provenance_postgresqldb: Dict[str, str],
				provenance_mongodb: Dict[str, str],
				aevisoUnsubmitted Not Done Inline Actions Not sure how to improve this to avoid repeating the test when a fixture that is not used gets a new instantiation. For instance, if `mongo` is selected we don't care about all possible instances for `provenance_postgresqldb` aeviso: Not sure how to improve this to avoid repeating the test when a fixture that is not used gets a…
	swh_rpc_client: RemoteProvenanceStorage,			swh_rpc_client: RemoteProvenanceStorage,
	) -> ProvenanceStorageInterface:			) -> ProvenanceStorageInterface:
	"""Return a working and initialized ProvenanceStorageInterface object"""			"""Return a working and initialized ProvenanceStorageInterface object"""

	if request.param == "remote":			if request.param == "remote":
	assert isinstance(swh_rpc_client, ProvenanceStorageInterface)			assert isinstance(swh_rpc_client, ProvenanceStorageInterface)
	return swh_rpc_client			return swh_rpc_client

				elif request.param == "mongodb":
				return get_provenance_storage(cls=request.param, db=provenance_mongodb)

	else:			else:
	# in test sessions, we DO want to raise any exception occurring at commit time			# in test sessions, we DO want to raise any exception occurring at commit time
	return get_provenance_storage(			return get_provenance_storage(
	cls=request.param, db=populated_db, raise_on_commit=True			cls=request.param, db=provenance_postgresqldb, raise_on_commit=True
	)			)


	provenance_postgresql = postgresql("postgresql_proc", dbname="provenance_tests")			provenance_postgresql = postgresql("postgresql_proc", dbname="provenance_tests")


	@pytest.fixture			@pytest.fixture
	def provenance(			def provenance(
	▲ Show 20 Lines • Show All 57 Lines • Show Last 20 Lines