diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,13 @@ *.csv +*.dump *.egg* *.log +*.mypy* *.prof *.zip *.orig *.bak -.coverage +.coverage* .tox .vscode __pycache__ diff --git a/pytest.ini b/pytest.ini --- a/pytest.ini +++ b/pytest.ini @@ -4,3 +4,8 @@ mongodb_fixture_dir = swh/provenance/tests/data/mongo mongodb_engine = mongomock mongodb_dbname = test + +postgresql_postgres_options = -N 500 + +rabbitmq_host = localhost +rabbitmq_port = 5672 diff --git a/requirements-test.txt b/requirements-test.txt --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,6 @@ pytest pytest-mongodb +pytest-rabbitmq swh.loader.git >= 0.8 swh.journal >= 0.8 types-Werkzeug diff --git a/swh/provenance/api/client.py b/swh/provenance/api/client.py --- a/swh/provenance/api/client.py +++ b/swh/provenance/api/client.py @@ -194,9 +194,11 @@ self._storage = get_provenance_storage(**storage_config) self._url = url + def open(self) -> None: self.start() while self._callback_queue is None: time.sleep(0.1) + self._storage.open() def close(self) -> None: assert self._connection is not None diff --git a/swh/provenance/api/server.py b/swh/provenance/api/server.py --- a/swh/provenance/api/server.py +++ b/swh/provenance/api/server.py @@ -672,9 +672,9 @@ self, binding_key: str, meth_name: str, relation: Optional[RelationType] ) -> None: storage = get_provenance_storage(**self._storage_config) + storage.open() request_queue = self._request_queues[binding_key] merge_items = ProvenanceStorageRabbitMQWorker.get_conflicts_func(meth_name) - while True: terminate = False elements = [] diff --git a/swh/provenance/tests/conftest.py b/swh/provenance/tests/conftest.py --- a/swh/provenance/tests/conftest.py +++ b/swh/provenance/tests/conftest.py @@ -66,7 +66,7 @@ return ProvenanceStorageRPCClient -@pytest.fixture(params=["mongodb", "postgresql", "rpcapi"]) +@pytest.fixture(params=["mongodb", "postgresql", "rpcapi", "rabbitmq"]) def provenance_storage( request: SubRequest, provenance_postgresqldb: Dict[str, str], @@ -94,6 +94,27 @@ yield mongodb_storage mongodb_storage.close() + elif request.param == "rabbitmq": + from swh.provenance.api.client import ProvenanceStorageRabbitMQClient + from swh.provenance.api.server import ProvenanceStorageRabbitMQServer + + request.getfixturevalue("rabbitmq") # loaded to ensure server is up + url = "amqp://guest:guest@localhost:5672/%2f" + storage_config = { + "cls": "postgresql", # TODO: also test with underlying mongodb storage + "db": provenance_postgresqldb, + "raise_on_commit": True, + } + server = ProvenanceStorageRabbitMQServer(url, storage_config) + client = ProvenanceStorageRabbitMQClient(url, storage_config) + assert isinstance(client, ProvenanceStorageInterface) + + server.start() + client.open() + yield client + client.close() + server.stop() + else: # in test sessions, we DO want to raise any exception occurring at commit time storage = get_provenance_storage( diff --git a/tox.ini b/tox.ini --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=black,flake8,mypy,py3 +envlist = black,flake8,mypy,py3 [testenv] extras = @@ -11,6 +11,7 @@ {envsitepackagesdir}/swh/provenance \ --cov={envsitepackagesdir}/swh/provenance \ --cov-branch {posargs} +passenv = HOME # required by pytest-rabbitmq [testenv:black] skip_install = true