Software Heritage provenance storage RPC server

diff --git a/requirements-test.txt b/requirements-test.txt --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,4 +3,3 @@ pytest-rabbitmq swh.loader.git >= 0.8 swh.journal >= 0.8 -types-Werkzeug diff --git a/swh/provenance/__init__.py b/swh/provenance/__init__.py --- a/swh/provenance/__init__.py +++ b/swh/provenance/__init__.py @@ -104,19 +104,4 @@ assert isinstance(rmq_storage, ProvenanceStorageInterface) return rmq_storage - elif cls in ["remote", "rpcapi"]: - from .api.client import ProvenanceStorageRPCClient - - if cls == "remote": - warnings.warn( - '"remote" class is deprecated for provenance storage, please ' - 'use "rpcapi" class instead.', - DeprecationWarning, - ) - - rpc_storage = ProvenanceStorageRPCClient(**kwargs) - if TYPE_CHECKING: - assert isinstance(rpc_storage, ProvenanceStorageInterface) - return rpc_storage - raise ValueError diff --git a/swh/provenance/api/client.py b/swh/provenance/api/client.py --- a/swh/provenance/api/client.py +++ b/swh/provenance/api/client.py @@ -18,7 +18,6 @@ import pika.frame import pika.spec -from swh.core.api import RPCClient from swh.core.api.serializers import encode_data_client as encode_data from swh.core.api.serializers import msgpack_loads as decode_data from swh.provenance import get_provenance_storage @@ -34,18 +33,6 @@ LOGGER = logging.getLogger(__name__) -class ProvenanceStorageRPCClient(RPCClient): - """Proxy to a remote provenance storage API""" - - backend_class = ProvenanceStorageInterface - extra_type_decoders = DECODERS - extra_type_encoders = ENCODERS - - -class ConfigurationError(Exception): - pass - - class ResponseTimeout(Exception): pass diff --git a/swh/provenance/api/server.py b/swh/provenance/api/server.py --- a/swh/provenance/api/server.py +++ b/swh/provenance/api/server.py @@ -23,10 +23,8 @@ from pika.exchange_type import ExchangeType import pika.frame import pika.spec -from werkzeug.routing import Rule from swh.core import config -from swh.core.api import JSONFormatter, MsgpackFormatter, RPCServerApp, negotiate from swh.core.api.serializers import encode_data_client as encode_data from swh.core.api.serializers import msgpack_loads as decode_data from swh.model.model import Sha1Git @@ -49,137 +47,6 @@ TERMINATE = object() -storage: Optional[ProvenanceStorageInterface] = None - - -def get_global_provenance_storage() -> ProvenanceStorageInterface: - global storage - if storage is None: - storage = get_provenance_storage(**app.config["provenance"]["storage"]) - return storage - - -class ProvenanceStorageRPCServerApp(RPCServerApp): - extra_type_decoders = DECODERS - extra_type_encoders = ENCODERS - - -app = ProvenanceStorageRPCServerApp( - __name__, - backend_class=ProvenanceStorageInterface, - backend_factory=get_global_provenance_storage, -) - - -def has_no_empty_params(rule: Rule) -> bool: - return len(rule.defaults or ()) >= len(rule.arguments or ()) - - -@app.route("/") -def index() -> str: - return """ -Software Heritage provenance storage RPC server - -

You have reached the -Software Heritage -provenance storage RPC server.
-See its -documentation -and API for more information

- -""" - - -@app.route("/site-map") -@negotiate(MsgpackFormatter) -@negotiate(JSONFormatter) -def site_map() -> List[Dict[str, Any]]: - links = [] - for rule in app.url_map.iter_rules(): - if has_no_empty_params(rule) and hasattr( - ProvenanceStorageInterface, rule.endpoint - ): - links.append( - dict( - rule=rule.rule, - description=getattr( - ProvenanceStorageInterface, rule.endpoint - ).__doc__, - ) - ) - # links is now a list of url, endpoint tuples - return links - - -def load_and_check_config( - config_path: Optional[str], type: str = "local" -) -> Dict[str, Any]: - """Check the minimal configuration is set to run the api or raise an - error explanation. - - Args: - config_path (str): Path to the configuration file to load - type (str): configuration type. For 'local' type, more - checks are done. - - Raises: - Error if the setup is not as expected - - Returns: - configuration as a dict - - """ - if config_path is None: - raise EnvironmentError("Configuration file must be defined") - - if not os.path.exists(config_path): - raise FileNotFoundError(f"Configuration file {config_path} does not exist") - - cfg = config.read(config_path) - - pcfg: Optional[Dict[str, Any]] = cfg.get("provenance") - if pcfg is None: - raise KeyError("Missing 'provenance' configuration") - - scfg: Optional[Dict[str, Any]] = pcfg.get("storage") - if scfg is None: - raise KeyError("Missing 'provenance.storage' configuration") - - if type == "local": - cls = scfg.get("cls") - if cls != "postgresql": - raise ValueError( - "The provenance backend can only be started with a 'postgresql' " - "configuration" - ) - - db = scfg.get("db") - if not db: - raise KeyError("Invalid configuration; missing 'db' config entry") - - return cfg - - -api_cfg: Optional[Dict[str, Any]] = None - - -def make_app_from_configfile() -> ProvenanceStorageRPCServerApp: - """Run the WSGI app from the webserver, loading the configuration from - a configuration file. - - SWH_CONFIG_FILENAME environment variable defines the - configuration path to load. - - """ - global api_cfg - if api_cfg is None: - config_path = os.environ.get("SWH_CONFIG_FILENAME") - api_cfg = load_and_check_config(config_path) - app.config.update(api_cfg) - handler = logging.StreamHandler() - app.logger.addHandler(handler) - return app - class ServerCommand(Enum): TERMINATE = "terminate" @@ -919,6 +786,55 @@ ) +def load_and_check_config( + config_path: Optional[str], type: str = "local" +) -> Dict[str, Any]: + """Check the minimal configuration is set to run the api or raise an + error explanation. + + Args: + config_path (str): Path to the configuration file to load + type (str): configuration type. For 'local' type, more + checks are done. + + Raises: + Error if the setup is not as expected + + Returns: + configuration as a dict + + """ + if config_path is None: + raise EnvironmentError("Configuration file must be defined") + + if not os.path.exists(config_path): + raise FileNotFoundError(f"Configuration file {config_path} does not exist") + + cfg = config.read(config_path) + + pcfg: Optional[Dict[str, Any]] = cfg.get("provenance") + if pcfg is None: + raise KeyError("Missing 'provenance' configuration") + + scfg: Optional[Dict[str, Any]] = pcfg.get("storage") + if scfg is None: + raise KeyError("Missing 'provenance.storage' configuration") + + if type == "local": + cls = scfg.get("cls") + if cls != "postgresql": + raise ValueError( + "The provenance backend can only be started with a 'postgresql' " + "configuration" + ) + + db = scfg.get("db") + if not db: + raise KeyError("Invalid configuration; missing 'db' config entry") + + return cfg + + def make_server_from_configfile() -> ProvenanceStorageRabbitMQServer: config_path = os.environ.get("SWH_CONFIG_FILENAME") server_cfg = load_and_check_config(config_path) diff --git a/swh/provenance/cli.py b/swh/provenance/cli.py --- a/swh/provenance/cli.py +++ b/swh/provenance/cli.py @@ -66,9 +66,6 @@ "dbname": "provenance", }, }, - # Remote RPC-API/PostgreSQL - # "cls": "rpcapi", - # "url": "http://localhost:8080/%2f", }, } } diff --git a/swh/provenance/tests/conftest.py b/swh/provenance/tests/conftest.py --- a/swh/provenance/tests/conftest.py +++ b/swh/provenance/tests/conftest.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta, timezone from os import path -from typing import Any, Dict, Generator, Iterable, Iterator +from typing import Any, Dict, Generator, Iterable from _pytest.fixtures import SubRequest import msgpack @@ -16,8 +16,6 @@ from swh.journal.serializers import msgpack_ext_hook from swh.provenance import get_provenance, get_provenance_storage -from swh.provenance.api.client import ProvenanceStorageRPCClient -import swh.provenance.api.server as server from swh.provenance.archive import ArchiveInterface from swh.provenance.interface import ProvenanceInterface, ProvenanceStorageInterface from swh.provenance.storage.archive import ArchiveStorage @@ -46,39 +44,15 @@ return postgresql.get_dsn_parameters() -# the Flask app used as server in these tests -@pytest.fixture -def app( - provenance_postgresqldb: Dict[str, str] -) -> Iterator[server.ProvenanceStorageRPCServerApp]: - assert hasattr(server, "storage") - server.storage = get_provenance_storage( - cls="postgresql", db=provenance_postgresqldb - ) - yield server.app - - -# the RPCClient class used as client used in these tests -@pytest.fixture -def swh_rpc_client_class() -> type: - return ProvenanceStorageRPCClient - - -@pytest.fixture(params=["mongodb", "postgresql", "rpcapi", "rabbitmq"]) +@pytest.fixture(params=["mongodb", "postgresql", "rabbitmq"]) def provenance_storage( request: SubRequest, provenance_postgresqldb: Dict[str, str], mongodb: pymongo.database.Database, - swh_rpc_client: ProvenanceStorageRPCClient, ) -> Generator[ProvenanceStorageInterface, None, None]: """Return a working and initialized ProvenanceStorageInterface object""" - if request.param == "rpcapi": - assert isinstance(swh_rpc_client, ProvenanceStorageInterface) - yield swh_rpc_client - swh_rpc_client.close() - - elif request.param == "mongodb": + if request.param == "mongodb": from swh.provenance.mongo.backend import ProvenanceStorageMongoDb # TODO: improve MongoDb initialization so that `get_provenance_storage` can be