Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9348318
D6273.id23062.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
10 KB
Subscribers
None
D6273.id23062.diff
View Options
diff --git a/requirements-test.txt b/requirements-test.txt
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -3,4 +3,3 @@
pytest-rabbitmq
swh.loader.git >= 0.8
swh.journal >= 0.8
-types-Werkzeug
diff --git a/swh/provenance/__init__.py b/swh/provenance/__init__.py
--- a/swh/provenance/__init__.py
+++ b/swh/provenance/__init__.py
@@ -104,19 +104,4 @@
assert isinstance(rmq_storage, ProvenanceStorageInterface)
return rmq_storage
- elif cls in ["remote", "rpcapi"]:
- from .api.client import ProvenanceStorageRPCClient
-
- if cls == "remote":
- warnings.warn(
- '"remote" class is deprecated for provenance storage, please '
- 'use "rpcapi" class instead.',
- DeprecationWarning,
- )
-
- rpc_storage = ProvenanceStorageRPCClient(**kwargs)
- if TYPE_CHECKING:
- assert isinstance(rpc_storage, ProvenanceStorageInterface)
- return rpc_storage
-
raise ValueError
diff --git a/swh/provenance/api/client.py b/swh/provenance/api/client.py
--- a/swh/provenance/api/client.py
+++ b/swh/provenance/api/client.py
@@ -18,7 +18,6 @@
import pika.frame
import pika.spec
-from swh.core.api import RPCClient
from swh.core.api.serializers import encode_data_client as encode_data
from swh.core.api.serializers import msgpack_loads as decode_data
from swh.provenance import get_provenance_storage
@@ -34,18 +33,6 @@
LOGGER = logging.getLogger(__name__)
-class ProvenanceStorageRPCClient(RPCClient):
- """Proxy to a remote provenance storage API"""
-
- backend_class = ProvenanceStorageInterface
- extra_type_decoders = DECODERS
- extra_type_encoders = ENCODERS
-
-
-class ConfigurationError(Exception):
- pass
-
-
class ResponseTimeout(Exception):
pass
diff --git a/swh/provenance/api/server.py b/swh/provenance/api/server.py
--- a/swh/provenance/api/server.py
+++ b/swh/provenance/api/server.py
@@ -23,10 +23,8 @@
from pika.exchange_type import ExchangeType
import pika.frame
import pika.spec
-from werkzeug.routing import Rule
from swh.core import config
-from swh.core.api import JSONFormatter, MsgpackFormatter, RPCServerApp, negotiate
from swh.core.api.serializers import encode_data_client as encode_data
from swh.core.api.serializers import msgpack_loads as decode_data
from swh.model.model import Sha1Git
@@ -49,137 +47,6 @@
TERMINATE = object()
-storage: Optional[ProvenanceStorageInterface] = None
-
-
-def get_global_provenance_storage() -> ProvenanceStorageInterface:
- global storage
- if storage is None:
- storage = get_provenance_storage(**app.config["provenance"]["storage"])
- return storage
-
-
-class ProvenanceStorageRPCServerApp(RPCServerApp):
- extra_type_decoders = DECODERS
- extra_type_encoders = ENCODERS
-
-
-app = ProvenanceStorageRPCServerApp(
- __name__,
- backend_class=ProvenanceStorageInterface,
- backend_factory=get_global_provenance_storage,
-)
-
-
-def has_no_empty_params(rule: Rule) -> bool:
- return len(rule.defaults or ()) >= len(rule.arguments or ())
-
-
-@app.route("/")
-def index() -> str:
- return """<html>
-<head><title>Software Heritage provenance storage RPC server</title></head>
-<body>
-<p>You have reached the
-<a href="https://www.softwareheritage.org/">Software Heritage</a>
-provenance storage RPC server.<br />
-See its
-<a href="https://docs.softwareheritage.org/devel/swh-provenance/">documentation
-and API</a> for more information</p>
-</body>
-</html>"""
-
-
-@app.route("/site-map")
-@negotiate(MsgpackFormatter)
-@negotiate(JSONFormatter)
-def site_map() -> List[Dict[str, Any]]:
- links = []
- for rule in app.url_map.iter_rules():
- if has_no_empty_params(rule) and hasattr(
- ProvenanceStorageInterface, rule.endpoint
- ):
- links.append(
- dict(
- rule=rule.rule,
- description=getattr(
- ProvenanceStorageInterface, rule.endpoint
- ).__doc__,
- )
- )
- # links is now a list of url, endpoint tuples
- return links
-
-
-def load_and_check_config(
- config_path: Optional[str], type: str = "local"
-) -> Dict[str, Any]:
- """Check the minimal configuration is set to run the api or raise an
- error explanation.
-
- Args:
- config_path (str): Path to the configuration file to load
- type (str): configuration type. For 'local' type, more
- checks are done.
-
- Raises:
- Error if the setup is not as expected
-
- Returns:
- configuration as a dict
-
- """
- if config_path is None:
- raise EnvironmentError("Configuration file must be defined")
-
- if not os.path.exists(config_path):
- raise FileNotFoundError(f"Configuration file {config_path} does not exist")
-
- cfg = config.read(config_path)
-
- pcfg: Optional[Dict[str, Any]] = cfg.get("provenance")
- if pcfg is None:
- raise KeyError("Missing 'provenance' configuration")
-
- scfg: Optional[Dict[str, Any]] = pcfg.get("storage")
- if scfg is None:
- raise KeyError("Missing 'provenance.storage' configuration")
-
- if type == "local":
- cls = scfg.get("cls")
- if cls != "postgresql":
- raise ValueError(
- "The provenance backend can only be started with a 'postgresql' "
- "configuration"
- )
-
- db = scfg.get("db")
- if not db:
- raise KeyError("Invalid configuration; missing 'db' config entry")
-
- return cfg
-
-
-api_cfg: Optional[Dict[str, Any]] = None
-
-
-def make_app_from_configfile() -> ProvenanceStorageRPCServerApp:
- """Run the WSGI app from the webserver, loading the configuration from
- a configuration file.
-
- SWH_CONFIG_FILENAME environment variable defines the
- configuration path to load.
-
- """
- global api_cfg
- if api_cfg is None:
- config_path = os.environ.get("SWH_CONFIG_FILENAME")
- api_cfg = load_and_check_config(config_path)
- app.config.update(api_cfg)
- handler = logging.StreamHandler()
- app.logger.addHandler(handler)
- return app
-
class ServerCommand(Enum):
TERMINATE = "terminate"
@@ -919,6 +786,55 @@
)
+def load_and_check_config(
+ config_path: Optional[str], type: str = "local"
+) -> Dict[str, Any]:
+ """Check the minimal configuration is set to run the api or raise an
+ error explanation.
+
+ Args:
+ config_path (str): Path to the configuration file to load
+ type (str): configuration type. For 'local' type, more
+ checks are done.
+
+ Raises:
+ Error if the setup is not as expected
+
+ Returns:
+ configuration as a dict
+
+ """
+ if config_path is None:
+ raise EnvironmentError("Configuration file must be defined")
+
+ if not os.path.exists(config_path):
+ raise FileNotFoundError(f"Configuration file {config_path} does not exist")
+
+ cfg = config.read(config_path)
+
+ pcfg: Optional[Dict[str, Any]] = cfg.get("provenance")
+ if pcfg is None:
+ raise KeyError("Missing 'provenance' configuration")
+
+ scfg: Optional[Dict[str, Any]] = pcfg.get("storage")
+ if scfg is None:
+ raise KeyError("Missing 'provenance.storage' configuration")
+
+ if type == "local":
+ cls = scfg.get("cls")
+ if cls != "postgresql":
+ raise ValueError(
+ "The provenance backend can only be started with a 'postgresql' "
+ "configuration"
+ )
+
+ db = scfg.get("db")
+ if not db:
+ raise KeyError("Invalid configuration; missing 'db' config entry")
+
+ return cfg
+
+
def make_server_from_configfile() -> ProvenanceStorageRabbitMQServer:
config_path = os.environ.get("SWH_CONFIG_FILENAME")
server_cfg = load_and_check_config(config_path)
diff --git a/swh/provenance/cli.py b/swh/provenance/cli.py
--- a/swh/provenance/cli.py
+++ b/swh/provenance/cli.py
@@ -66,9 +66,6 @@
"dbname": "provenance",
},
},
- # Remote RPC-API/PostgreSQL
- # "cls": "rpcapi",
- # "url": "http://localhost:8080/%2f",
},
}
}
diff --git a/swh/provenance/tests/conftest.py b/swh/provenance/tests/conftest.py
--- a/swh/provenance/tests/conftest.py
+++ b/swh/provenance/tests/conftest.py
@@ -5,7 +5,7 @@
from datetime import datetime, timedelta, timezone
from os import path
-from typing import Any, Dict, Generator, Iterable, Iterator
+from typing import Any, Dict, Generator, Iterable
from _pytest.fixtures import SubRequest
import msgpack
@@ -16,8 +16,6 @@
from swh.journal.serializers import msgpack_ext_hook
from swh.provenance import get_provenance, get_provenance_storage
-from swh.provenance.api.client import ProvenanceStorageRPCClient
-import swh.provenance.api.server as server
from swh.provenance.archive import ArchiveInterface
from swh.provenance.interface import ProvenanceInterface, ProvenanceStorageInterface
from swh.provenance.storage.archive import ArchiveStorage
@@ -46,39 +44,15 @@
return postgresql.get_dsn_parameters()
-# the Flask app used as server in these tests
-@pytest.fixture
-def app(
- provenance_postgresqldb: Dict[str, str]
-) -> Iterator[server.ProvenanceStorageRPCServerApp]:
- assert hasattr(server, "storage")
- server.storage = get_provenance_storage(
- cls="postgresql", db=provenance_postgresqldb
- )
- yield server.app
-
-
-# the RPCClient class used as client used in these tests
-@pytest.fixture
-def swh_rpc_client_class() -> type:
- return ProvenanceStorageRPCClient
-
-
-@pytest.fixture(params=["mongodb", "postgresql", "rpcapi", "rabbitmq"])
+@pytest.fixture(params=["mongodb", "postgresql", "rabbitmq"])
def provenance_storage(
request: SubRequest,
provenance_postgresqldb: Dict[str, str],
mongodb: pymongo.database.Database,
- swh_rpc_client: ProvenanceStorageRPCClient,
) -> Generator[ProvenanceStorageInterface, None, None]:
"""Return a working and initialized ProvenanceStorageInterface object"""
- if request.param == "rpcapi":
- assert isinstance(swh_rpc_client, ProvenanceStorageInterface)
- yield swh_rpc_client
- swh_rpc_client.close()
-
- elif request.param == "mongodb":
+ if request.param == "mongodb":
from swh.provenance.mongo.backend import ProvenanceStorageMongoDb
# TODO: improve MongoDb initialization so that `get_provenance_storage` can be
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Jul 3 2025, 6:23 PM (5 w, 3 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3227372
Attached To
D6273: Remove remote storage based on `swh.core.api.RPCClient`
Event Timeline
Log In to Comment