diff --git a/Makefile.local b/Makefile.local index 1181cea..64f74b9 100644 --- a/Makefile.local +++ b/Makefile.local @@ -1,17 +1,17 @@ POM_PATH=java/pom.xml java: mvn -f $(POM_PATH) compile assembly:single java-doc: mvn -f $(POM_PATH) javadoc:javadoc java-%: mvn -f $(POM_PATH) $* protoc: - python -m grpc_tools.protoc -I. --python_out=. --mypy_out=. --grpc_python_out=. swh/graph/rpc/*.proto + python3 -m grpc_tools.protoc -I. --python_out=. --mypy_out=. --grpc_python_out=. swh/graph/grpc/*.proto clean-java: java-clean .PHONY: java clean-java diff --git a/PKG-INFO b/PKG-INFO index c5989bd..0eb9464 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,52 +1,52 @@ Metadata-Version: 2.1 Name: swh.graph -Version: 1.0.2 +Version: 2.0.0 Summary: Software Heritage graph service Home-page: https://forge.softwareheritage.org/diffusion/DGRPH Author: Software Heritage developers Author-email: swh-devel@inria.fr Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-graph Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-graph/ Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 3 - Alpha Requires-Python: >=3.7 Description-Content-Type: text/x-rst Provides-Extra: testing License-File: LICENSE License-File: AUTHORS Software Heritage - graph service ================================= Tooling and services, collectively known as ``swh-graph``, providing fast access to the graph representation of the `Software Heritage `_ `archive `_. The service is in-memory, based on a compressed representation of the Software Heritage Merkle DAG. Bibliography ------------ In addition to accompanying technical documentation, ``swh-graph`` is also described in the following scientific paper. If you publish results based on ``swh-graph``, please acknowledge it by citing the paper as follows: .. note:: Paolo Boldi, Antoine Pietri, Sebastiano Vigna, Stefano Zacchiroli. `Ultra-Large-Scale Repository Analysis via Graph Compression `_. In proceedings of `SANER 2020 `_: The 27th IEEE International Conference on Software Analysis, Evolution and Reengineering, pages 184-194. IEEE 2020. Links: `preprint `_, `bibtex `_. diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..ed9e52e --- /dev/null +++ b/conftest.py @@ -0,0 +1,8 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +pytest_plugins = [ + "swh.graph.pytest_plugin", +] diff --git a/mypy.ini b/mypy.ini index 147a027..dffcb6e 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,21 +1,21 @@ [mypy] namespace_packages = True warn_unused_ignores = True exclude = (?x)( - ^swh/graph/rpc + ^swh/graph/grpc ) # 3rd party libraries without stubs (yet) [mypy-pkg_resources.*] ignore_missing_imports = True [mypy-psutil.*] ignore_missing_imports = True [mypy-py4j.*] ignore_missing_imports = True [mypy-pytest.*] ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index 8c8af87..5c4d3b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,16 +1,16 @@ [tool.black] target-version = ['py37'] extend-exclude = ''' /( - | swh/graph/rpc + | swh/graph/grpc )/ ''' [tool.isort] multi_line_output = 3 include_trailing_comma = true force_grid_wrap = 0 use_parentheses = true ensure_newline_before_comments = true line_length = 88 force_sort_within_sections = true diff --git a/setup.cfg b/setup.cfg index b3eac4d..187f489 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,10 +1,10 @@ [flake8] select = C,E,F,W,B950 ignore = E203,E231,E501,W503 max-line-length = 88 -extend_exclude = swh/graph/rpc +extend_exclude = swh/graph/grpc [egg_info] tag_build = tag_date = 0 diff --git a/swh.graph.egg-info/PKG-INFO b/swh.graph.egg-info/PKG-INFO index c5989bd..0eb9464 100644 --- a/swh.graph.egg-info/PKG-INFO +++ b/swh.graph.egg-info/PKG-INFO @@ -1,52 +1,52 @@ Metadata-Version: 2.1 Name: swh.graph -Version: 1.0.2 +Version: 2.0.0 Summary: Software Heritage graph service Home-page: https://forge.softwareheritage.org/diffusion/DGRPH Author: Software Heritage developers Author-email: swh-devel@inria.fr Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-graph Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-graph/ Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 3 - Alpha Requires-Python: >=3.7 Description-Content-Type: text/x-rst Provides-Extra: testing License-File: LICENSE License-File: AUTHORS Software Heritage - graph service ================================= Tooling and services, collectively known as ``swh-graph``, providing fast access to the graph representation of the `Software Heritage `_ `archive `_. The service is in-memory, based on a compressed representation of the Software Heritage Merkle DAG. Bibliography ------------ In addition to accompanying technical documentation, ``swh-graph`` is also described in the following scientific paper. If you publish results based on ``swh-graph``, please acknowledge it by citing the paper as follows: .. note:: Paolo Boldi, Antoine Pietri, Sebastiano Vigna, Stefano Zacchiroli. `Ultra-Large-Scale Repository Analysis via Graph Compression `_. In proceedings of `SANER 2020 `_: The 27th IEEE International Conference on Software Analysis, Evolution and Reengineering, pages 184-194. IEEE 2020. Links: `preprint `_, `bibtex `_. diff --git a/swh.graph.egg-info/SOURCES.txt b/swh.graph.egg-info/SOURCES.txt index bf77475..48d4758 100644 --- a/swh.graph.egg-info/SOURCES.txt +++ b/swh.graph.egg-info/SOURCES.txt @@ -1,258 +1,259 @@ .git-blame-ignore-revs .gitignore .pre-commit-config.yaml AUTHORS CODE_OF_CONDUCT.md CONTRIBUTORS LICENSE MANIFEST.in Makefile Makefile.local README.rst +conftest.py mypy.ini pyproject.toml pytest.ini requirements-swh.txt requirements-test.txt requirements.txt setup.cfg setup.py tox.ini docker/Dockerfile docker/build.sh docker/run.sh docs/.gitignore docs/Makefile docs/Makefile.local docs/README.rst docs/api.rst docs/cli.rst docs/compression.rst docs/conf.py docs/docker.rst docs/git2graph.md docs/grpc-api.rst docs/index.rst docs/java-api.rst docs/memory.rst docs/quickstart.rst docs/_static/.placeholder docs/_templates/.placeholder docs/images/.gitignore docs/images/Makefile docs/images/compression_steps.dot java/.coding-style.xml java/.gitignore java/AUTHORS java/LICENSE java/README.md java/pom.xml java/.mvn/jvm.config java/src/main/proto java/src/main/java/org/softwareheritage/graph/AllowedEdges.java java/src/main/java/org/softwareheritage/graph/AllowedNodes.java java/src/main/java/org/softwareheritage/graph/SWHID.java java/src/main/java/org/softwareheritage/graph/Subgraph.java java/src/main/java/org/softwareheritage/graph/SwhBidirectionalGraph.java java/src/main/java/org/softwareheritage/graph/SwhGraph.java java/src/main/java/org/softwareheritage/graph/SwhGraphProperties.java java/src/main/java/org/softwareheritage/graph/SwhType.java java/src/main/java/org/softwareheritage/graph/SwhUnidirectionalGraph.java java/src/main/java/org/softwareheritage/graph/compress/CSVEdgeDataset.java java/src/main/java/org/softwareheritage/graph/compress/ComposePermutations.java java/src/main/java/org/softwareheritage/graph/compress/ExtractNodes.java java/src/main/java/org/softwareheritage/graph/compress/ExtractPersons.java java/src/main/java/org/softwareheritage/graph/compress/GraphDataset.java java/src/main/java/org/softwareheritage/graph/compress/LabelMapBuilder.java java/src/main/java/org/softwareheritage/graph/compress/NodeMapBuilder.java java/src/main/java/org/softwareheritage/graph/compress/ORCGraphDataset.java java/src/main/java/org/softwareheritage/graph/compress/ScatteredArcsORCGraph.java java/src/main/java/org/softwareheritage/graph/compress/WriteNodeProperties.java java/src/main/java/org/softwareheritage/graph/experiments/forks/ForkCC.java java/src/main/java/org/softwareheritage/graph/experiments/forks/ForkCliques.java java/src/main/java/org/softwareheritage/graph/experiments/forks/ListEmptyOrigins.java java/src/main/java/org/softwareheritage/graph/experiments/topology/AveragePaths.java java/src/main/java/org/softwareheritage/graph/experiments/topology/ClusteringCoefficient.java java/src/main/java/org/softwareheritage/graph/experiments/topology/ConnectedComponents.java java/src/main/java/org/softwareheritage/graph/experiments/topology/InOutDegree.java java/src/main/java/org/softwareheritage/graph/experiments/topology/SubdatasetSizeFunction.java java/src/main/java/org/softwareheritage/graph/labels/DirEntry.java java/src/main/java/org/softwareheritage/graph/labels/SwhLabel.java java/src/main/java/org/softwareheritage/graph/maps/NodeIdMap.java java/src/main/java/org/softwareheritage/graph/maps/NodeTypesMap.java java/src/main/java/org/softwareheritage/graph/rpc/GraphServer.java java/src/main/java/org/softwareheritage/graph/rpc/NodePropertyBuilder.java java/src/main/java/org/softwareheritage/graph/rpc/Traversal.java java/src/main/java/org/softwareheritage/graph/utils/DumpProperties.java java/src/main/java/org/softwareheritage/graph/utils/ExportSubdataset.java java/src/main/java/org/softwareheritage/graph/utils/FindEarliestRevision.java java/src/main/java/org/softwareheritage/graph/utils/ForkJoinBigQuickSort2.java java/src/main/java/org/softwareheritage/graph/utils/ForkJoinQuickSort3.java java/src/main/java/org/softwareheritage/graph/utils/MPHTranslate.java java/src/main/java/org/softwareheritage/graph/utils/ReadGraph.java java/src/main/java/org/softwareheritage/graph/utils/ReadLabelledGraph.java java/src/main/java/org/softwareheritage/graph/utils/Sort.java java/src/test/java/org/softwareheritage/graph/AllowedEdgesTest.java java/src/test/java/org/softwareheritage/graph/AllowedNodesTest.java java/src/test/java/org/softwareheritage/graph/GraphTest.java java/src/test/java/org/softwareheritage/graph/SubgraphTest.java java/src/test/java/org/softwareheritage/graph/compress/ExtractNodesTest.java java/src/test/java/org/softwareheritage/graph/compress/ExtractPersonsTest.java java/src/test/java/org/softwareheritage/graph/rpc/CountEdgesTest.java java/src/test/java/org/softwareheritage/graph/rpc/CountNodesTest.java java/src/test/java/org/softwareheritage/graph/rpc/FindPathBetweenTest.java java/src/test/java/org/softwareheritage/graph/rpc/FindPathToTest.java java/src/test/java/org/softwareheritage/graph/rpc/GetNodeTest.java java/src/test/java/org/softwareheritage/graph/rpc/StatsTest.java java/src/test/java/org/softwareheritage/graph/rpc/TraversalServiceTest.java java/src/test/java/org/softwareheritage/graph/rpc/TraverseLeavesTest.java java/src/test/java/org/softwareheritage/graph/rpc/TraverseNeighborsTest.java java/src/test/java/org/softwareheritage/graph/rpc/TraverseNodesPropertiesTest.java java/src/test/java/org/softwareheritage/graph/rpc/TraverseNodesTest.java java/src/test/java/org/softwareheritage/graph/utils/ForkJoinBigQuickSort2Test.java java/src/test/java/org/softwareheritage/graph/utils/ForkJoinQuickSort3Test.java -java/target/swh-graph-1.0.2.jar +java/target/swh-graph-2.0.0.jar proto/swhgraph.proto reports/.gitignore reports/benchmarks/Makefile reports/benchmarks/benchmarks.tex reports/experiments/Makefile reports/experiments/experiments.tex reports/linux_log/LinuxLog.java reports/linux_log/Makefile reports/linux_log/linux_log.tex reports/node_mapping/Makefile reports/node_mapping/NodeIdMapHaloDB.java reports/node_mapping/NodeIdMapRocksDB.java reports/node_mapping/node_mapping.tex swh/__init__.py swh.graph.egg-info/PKG-INFO swh.graph.egg-info/SOURCES.txt swh.graph.egg-info/dependency_links.txt swh.graph.egg-info/entry_points.txt swh.graph.egg-info/requires.txt swh.graph.egg-info/top_level.txt swh/graph/__init__.py swh/graph/cli.py swh/graph/client.py swh/graph/config.py +swh/graph/grpc_server.py swh/graph/http_client.py swh/graph/http_naive_client.py -swh/graph/http_server.py +swh/graph/http_rpc_server.py swh/graph/naive_client.py swh/graph/py.typed -swh/graph/rpc_server.py +swh/graph/pytest_plugin.py swh/graph/webgraph.py -swh/graph/rpc/swhgraph.proto -swh/graph/rpc/swhgraph_pb2.py -swh/graph/rpc/swhgraph_pb2.pyi -swh/graph/rpc/swhgraph_pb2_grpc.py +swh/graph/grpc/swhgraph.proto +swh/graph/grpc/swhgraph_pb2.py +swh/graph/grpc/swhgraph_pb2.pyi +swh/graph/grpc/swhgraph_pb2_grpc.py swh/graph/tests/__init__.py -swh/graph/tests/conftest.py swh/graph/tests/test_cli.py swh/graph/tests/test_grpc.py swh/graph/tests/test_http_client.py swh/graph/tests/test_http_server_down.py swh/graph/tests/dataset/generate_dataset.py swh/graph/tests/dataset/compressed/example-labelled.labeloffsets swh/graph/tests/dataset/compressed/example-labelled.labels swh/graph/tests/dataset/compressed/example-labelled.properties swh/graph/tests/dataset/compressed/example-transposed-labelled.labeloffsets swh/graph/tests/dataset/compressed/example-transposed-labelled.labels swh/graph/tests/dataset/compressed/example-transposed-labelled.properties swh/graph/tests/dataset/compressed/example-transposed.graph swh/graph/tests/dataset/compressed/example-transposed.obl swh/graph/tests/dataset/compressed/example-transposed.offsets swh/graph/tests/dataset/compressed/example-transposed.properties swh/graph/tests/dataset/compressed/example.edges.count.txt swh/graph/tests/dataset/compressed/example.edges.stats.txt swh/graph/tests/dataset/compressed/example.graph swh/graph/tests/dataset/compressed/example.indegree swh/graph/tests/dataset/compressed/example.labels.count.txt swh/graph/tests/dataset/compressed/example.labels.csv.zst swh/graph/tests/dataset/compressed/example.labels.fcl.bytearray swh/graph/tests/dataset/compressed/example.labels.fcl.pointers swh/graph/tests/dataset/compressed/example.labels.fcl.properties swh/graph/tests/dataset/compressed/example.labels.mph swh/graph/tests/dataset/compressed/example.mph swh/graph/tests/dataset/compressed/example.node2swhid.bin swh/graph/tests/dataset/compressed/example.node2type.map swh/graph/tests/dataset/compressed/example.nodes.count.txt swh/graph/tests/dataset/compressed/example.nodes.csv.zst swh/graph/tests/dataset/compressed/example.nodes.stats.txt swh/graph/tests/dataset/compressed/example.obl swh/graph/tests/dataset/compressed/example.offsets swh/graph/tests/dataset/compressed/example.order swh/graph/tests/dataset/compressed/example.outdegree swh/graph/tests/dataset/compressed/example.persons.count.txt swh/graph/tests/dataset/compressed/example.persons.csv.zst swh/graph/tests/dataset/compressed/example.persons.mph swh/graph/tests/dataset/compressed/example.properties swh/graph/tests/dataset/compressed/example.property.author_id.bin swh/graph/tests/dataset/compressed/example.property.author_timestamp.bin swh/graph/tests/dataset/compressed/example.property.author_timestamp_offset.bin swh/graph/tests/dataset/compressed/example.property.committer_id.bin swh/graph/tests/dataset/compressed/example.property.committer_timestamp.bin swh/graph/tests/dataset/compressed/example.property.committer_timestamp_offset.bin swh/graph/tests/dataset/compressed/example.property.content.is_skipped.bin swh/graph/tests/dataset/compressed/example.property.content.length.bin swh/graph/tests/dataset/compressed/example.property.message.bin swh/graph/tests/dataset/compressed/example.property.message.offset.bin swh/graph/tests/dataset/compressed/example.property.tag_name.bin swh/graph/tests/dataset/compressed/example.property.tag_name.offset.bin swh/graph/tests/dataset/compressed/example.stats swh/graph/tests/dataset/edges/content/graph-all.edges.csv.zst swh/graph/tests/dataset/edges/content/graph-all.nodes.csv.zst swh/graph/tests/dataset/edges/directory/graph-all.edges.csv.zst swh/graph/tests/dataset/edges/directory/graph-all.nodes.csv.zst swh/graph/tests/dataset/edges/origin/graph-all.edges.csv.zst swh/graph/tests/dataset/edges/origin/graph-all.nodes.csv.zst swh/graph/tests/dataset/edges/release/graph-all.edges.csv.zst swh/graph/tests/dataset/edges/release/graph-all.nodes.csv.zst swh/graph/tests/dataset/edges/revision/graph-all.edges.csv.zst swh/graph/tests/dataset/edges/revision/graph-all.nodes.csv.zst swh/graph/tests/dataset/edges/snapshot/graph-all.edges.csv.zst swh/graph/tests/dataset/edges/snapshot/graph-all.nodes.csv.zst swh/graph/tests/dataset/img/.gitignore swh/graph/tests/dataset/img/Makefile swh/graph/tests/dataset/img/example.dot swh/graph/tests/dataset/orc/content/content-all.orc swh/graph/tests/dataset/orc/directory/directory-all.orc swh/graph/tests/dataset/orc/directory_entry/directory_entry-all.orc swh/graph/tests/dataset/orc/origin/origin-all.orc swh/graph/tests/dataset/orc/origin_visit/origin_visit-all.orc swh/graph/tests/dataset/orc/origin_visit_status/origin_visit_status-all.orc swh/graph/tests/dataset/orc/release/release-all.orc swh/graph/tests/dataset/orc/revision/revision-all.orc swh/graph/tests/dataset/orc/revision_extra_headers/revision_extra_headers-all.orc swh/graph/tests/dataset/orc/revision_history/revision_history-all.orc swh/graph/tests/dataset/orc/skipped_content/skipped_content-all.orc swh/graph/tests/dataset/orc/snapshot/snapshot-all.orc swh/graph/tests/dataset/orc/snapshot_branch/snapshot_branch-all.orc tools/dir2graph tools/swhid2int2int2swhid.sh tools/git2graph/.gitignore tools/git2graph/Makefile tools/git2graph/README.md tools/git2graph/git2graph.c tools/git2graph/tests/edge-filters.bats tools/git2graph/tests/full-graph.bats tools/git2graph/tests/node-filters.bats tools/git2graph/tests/repo_helper.bash tools/git2graph/tests/data/sample-repo.tgz tools/git2graph/tests/data/graphs/dir-nodes/edges.csv tools/git2graph/tests/data/graphs/dir-nodes/nodes.csv tools/git2graph/tests/data/graphs/from-dir-edges/edges.csv tools/git2graph/tests/data/graphs/from-dir-edges/nodes.csv tools/git2graph/tests/data/graphs/from-rel-edges/edges.csv tools/git2graph/tests/data/graphs/from-rel-edges/nodes.csv tools/git2graph/tests/data/graphs/fs-nodes/edges.csv tools/git2graph/tests/data/graphs/fs-nodes/nodes.csv tools/git2graph/tests/data/graphs/full/edges.csv tools/git2graph/tests/data/graphs/full/nodes.csv tools/git2graph/tests/data/graphs/rev-edges/edges.csv tools/git2graph/tests/data/graphs/rev-edges/nodes.csv tools/git2graph/tests/data/graphs/rev-nodes/edges.csv tools/git2graph/tests/data/graphs/rev-nodes/nodes.csv tools/git2graph/tests/data/graphs/to-rev-edges/edges.csv tools/git2graph/tests/data/graphs/to-rev-edges/nodes.csv \ No newline at end of file diff --git a/swh/graph/cli.py b/swh/graph/cli.py index 9eaf547..67d8241 100644 --- a/swh/graph/cli.py +++ b/swh/graph/cli.py @@ -1,200 +1,247 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, Set, Tuple # WARNING: do not import unnecessary things here to keep cli startup time under # control import click from swh.core.cli import CONTEXT_SETTINGS, AliasedGroup from swh.core.cli import swh as swh_cli_group if TYPE_CHECKING: from swh.graph.webgraph import CompressionStep # noqa class StepOption(click.ParamType): """click type for specifying a compression step on the CLI parse either individual steps, specified as step names or integers, or step ranges """ name = "compression step" def convert(self, value, param, ctx): # type: (...) -> Set[CompressionStep] from swh.graph.webgraph import COMP_SEQ, CompressionStep # noqa steps: Set[CompressionStep] = set() specs = value.split(",") for spec in specs: if "-" in spec: # step range (raw_l, raw_r) = spec.split("-", maxsplit=1) if raw_l == "": # no left endpoint raw_l = COMP_SEQ[0].name if raw_r == "": # no right endpoint raw_r = COMP_SEQ[-1].name l_step = self.convert(raw_l, param, ctx) r_step = self.convert(raw_r, param, ctx) if len(l_step) != 1 or len(r_step) != 1: self.fail(f"invalid step specification: {value}, " f"see --help") l_idx = l_step.pop() r_idx = r_step.pop() steps = steps.union( set(CompressionStep(i) for i in range(l_idx.value, r_idx.value + 1)) ) else: # singleton step try: steps.add(CompressionStep(int(spec))) # integer step except ValueError: try: steps.add(CompressionStep[spec.upper()]) # step name except KeyError: self.fail( f"invalid step specification: {value}, " f"see --help" ) return steps class PathlibPath(click.Path): """A Click path argument that returns a pathlib Path, not a string""" def convert(self, value, param, ctx): return Path(super().convert(value, param, ctx)) DEFAULT_CONFIG: Dict[str, Tuple[str, Any]] = {"graph": ("dict", {})} @swh_cli_group.group(name="graph", context_settings=CONTEXT_SETTINGS, cls=AliasedGroup) @click.option( "--config-file", "-C", default=None, type=click.Path( exists=True, dir_okay=False, ), help="YAML configuration file", ) @click.pass_context def graph_cli_group(ctx, config_file): """Software Heritage graph tools.""" from swh.core import config ctx.ensure_object(dict) conf = config.read(config_file, DEFAULT_CONFIG) if "graph" not in conf: raise ValueError( 'no "graph" stanza found in configuration file %s' % config_file ) ctx.obj["config"] = conf @graph_cli_group.command(name="rpc-serve") @click.option( "--host", "-h", default="0.0.0.0", metavar="IP", show_default=True, help="host IP address to bind the server on", ) @click.option( "--port", "-p", default=5009, type=click.INT, metavar="PORT", show_default=True, help="port to bind the server on", ) @click.option( "--graph", "-g", required=True, metavar="GRAPH", help="compressed graph basename" ) @click.pass_context def serve(ctx, host, port, graph): """run the graph RPC service""" import aiohttp.web - from swh.graph.http_server import make_app + from swh.graph.http_rpc_server import make_app config = ctx.obj["config"] config.setdefault("graph", {}) config["graph"]["path"] = graph app = make_app(config=config) aiohttp.web.run_app(app, host=host, port=port) +@graph_cli_group.command(name="grpc-serve") +@click.option( + "--port", + "-p", + default=50091, + type=click.INT, + metavar="PORT", + show_default=True, + help=( + "port to bind the server on (note: host is not configurable " + "for now and will be 0.0.0.0)" + ), +) +@click.option( + "--java-home", + "-j", + default=None, + metavar="JAVA_HOME", + help="absolute path to the Java Runtime Environment (JRE)", +) +@click.option( + "--graph", "-g", required=True, metavar="GRAPH", help="compressed graph basename" +) +@click.pass_context +def grpc_serve(ctx, host, port, java_home, graph): + """start the graph GRPC service + + This command uses execve to execute the java GRPC service. + """ + import os + from pathlib import Path + + from swh.graph.grpc_server import build_grpc_server_cmdline + + config = ctx.obj["config"] + config.setdefault("graph", {}) + config["graph"]["path"] = graph + cmd, port = build_grpc_server_cmdline(**config["graph"]) + + java_bin = cmd[0] + if java_home is not None: + java_bin = str(Path(java_home) / "bin" / java_bin) + + print(f"Starting the GRPC server on 0.0.0.0:{port}") + os.execvp(java_bin, cmd) + + @graph_cli_group.command() @click.option( "--input-dataset", "-i", required=True, type=PathlibPath(), help="graph dataset directory, in ORC format", ) @click.option( "--output-directory", "-o", required=True, type=PathlibPath(), help="directory where to store compressed graph", ) @click.option( "--graph-name", "-g", default="graph", metavar="NAME", help="name of the output graph (default: 'graph')", ) @click.option( "--steps", "-s", metavar="STEPS", type=StepOption(), help="run only these compression steps (default: all steps)", ) @click.pass_context def compress(ctx, input_dataset, output_directory, graph_name, steps): """Compress a graph using WebGraph Input: a directory containing a graph dataset in ORC format Output: a directory containing a WebGraph compressed graph Compression steps are: (1) extract_nodes, (2) mph, (3) bv, (4) bfs, (5) permute_bfs, (6) transpose_bfs, (7) simplify, (8) llp, (9) permute_llp, (10) obl, (11) compose_orders, (12) stats, (13) transpose, (14) transpose_obl, (15) maps, (16) extract_persons, (17) mph_persons, (18) node_properties, (19) mph_labels, (20) fcl_labels, (21) edge_labels, (22) edge_labels_obl, (23) edge_labels_transpose_obl, (24) clean_tmp. Compression steps can be selected by name or number using --steps, separating them with commas; step ranges (e.g., 3-9, 6-, etc.) are also supported. """ from swh.graph import webgraph try: conf = ctx.obj["config"]["graph"]["compress"] except KeyError: conf = {} # use defaults webgraph.compress(graph_name, input_dataset, output_directory, steps, conf) def main(): return graph_cli_group(auto_envvar_prefix="SWH_GRAPH") if __name__ == "__main__": main() diff --git a/swh/graph/rpc/swhgraph.proto b/swh/graph/grpc/swhgraph.proto similarity index 100% rename from swh/graph/rpc/swhgraph.proto rename to swh/graph/grpc/swhgraph.proto diff --git a/swh/graph/grpc/swhgraph_pb2.py b/swh/graph/grpc/swhgraph_pb2.py new file mode 100644 index 0000000..2809a58 --- /dev/null +++ b/swh/graph/grpc/swhgraph_pb2.py @@ -0,0 +1,196 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: swh/graph/grpc/swhgraph.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import field_mask_pb2 as google_dot_protobuf_dot_field__mask__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1dswh/graph/grpc/swhgraph.proto\x12\tswh.graph\x1a google/protobuf/field_mask.proto\"W\n\x0eGetNodeRequest\x12\r\n\x05swhid\x18\x01 \x01(\t\x12-\n\x04mask\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x00\x88\x01\x01\x42\x07\n\x05_mask\"\xd8\x02\n\x10TraversalRequest\x12\x0b\n\x03src\x18\x01 \x03(\t\x12,\n\tdirection\x18\x02 \x01(\x0e\x32\x19.swh.graph.GraphDirection\x12\x12\n\x05\x65\x64ges\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tmax_edges\x18\x04 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\tmin_depth\x18\x05 \x01(\x03H\x02\x88\x01\x01\x12\x16\n\tmax_depth\x18\x06 \x01(\x03H\x03\x88\x01\x01\x12\x30\n\x0creturn_nodes\x18\x07 \x01(\x0b\x32\x15.swh.graph.NodeFilterH\x04\x88\x01\x01\x12-\n\x04mask\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x05\x88\x01\x01\x42\x08\n\x06_edgesB\x0c\n\n_max_edgesB\x0c\n\n_min_depthB\x0c\n\n_max_depthB\x0f\n\r_return_nodesB\x07\n\x05_mask\"\x97\x02\n\x11\x46indPathToRequest\x12\x0b\n\x03src\x18\x01 \x03(\t\x12%\n\x06target\x18\x02 \x01(\x0b\x32\x15.swh.graph.NodeFilter\x12,\n\tdirection\x18\x03 \x01(\x0e\x32\x19.swh.graph.GraphDirection\x12\x12\n\x05\x65\x64ges\x18\x04 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tmax_edges\x18\x05 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\tmax_depth\x18\x06 \x01(\x03H\x02\x88\x01\x01\x12-\n\x04mask\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x03\x88\x01\x01\x42\x08\n\x06_edgesB\x0c\n\n_max_edgesB\x0c\n\n_max_depthB\x07\n\x05_mask\"\x81\x03\n\x16\x46indPathBetweenRequest\x12\x0b\n\x03src\x18\x01 \x03(\t\x12\x0b\n\x03\x64st\x18\x02 \x03(\t\x12,\n\tdirection\x18\x03 \x01(\x0e\x32\x19.swh.graph.GraphDirection\x12\x39\n\x11\x64irection_reverse\x18\x04 \x01(\x0e\x32\x19.swh.graph.GraphDirectionH\x00\x88\x01\x01\x12\x12\n\x05\x65\x64ges\x18\x05 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\redges_reverse\x18\x06 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tmax_edges\x18\x07 \x01(\x03H\x03\x88\x01\x01\x12\x16\n\tmax_depth\x18\x08 \x01(\x03H\x04\x88\x01\x01\x12-\n\x04mask\x18\t \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x05\x88\x01\x01\x42\x14\n\x12_direction_reverseB\x08\n\x06_edgesB\x10\n\x0e_edges_reverseB\x0c\n\n_max_edgesB\x0c\n\n_max_depthB\x07\n\x05_mask\"\xb2\x01\n\nNodeFilter\x12\x12\n\x05types\x18\x01 \x01(\tH\x00\x88\x01\x01\x12%\n\x18min_traversal_successors\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12%\n\x18max_traversal_successors\x18\x03 \x01(\x03H\x02\x88\x01\x01\x42\x08\n\x06_typesB\x1b\n\x19_min_traversal_successorsB\x1b\n\x19_max_traversal_successors\"\x92\x02\n\x04Node\x12\r\n\x05swhid\x18\x01 \x01(\t\x12\'\n\tsuccessor\x18\x02 \x03(\x0b\x32\x14.swh.graph.Successor\x12\x1b\n\x0enum_successors\x18\t \x01(\x03H\x01\x88\x01\x01\x12%\n\x03\x63nt\x18\x03 \x01(\x0b\x32\x16.swh.graph.ContentDataH\x00\x12&\n\x03rev\x18\x05 \x01(\x0b\x32\x17.swh.graph.RevisionDataH\x00\x12%\n\x03rel\x18\x06 \x01(\x0b\x32\x16.swh.graph.ReleaseDataH\x00\x12$\n\x03ori\x18\x08 \x01(\x0b\x32\x15.swh.graph.OriginDataH\x00\x42\x06\n\x04\x64\x61taB\x11\n\x0f_num_successors\"U\n\x04Path\x12\x1d\n\x04node\x18\x01 \x03(\x0b\x32\x0f.swh.graph.Node\x12\x1b\n\x0emidpoint_index\x18\x02 \x01(\x05H\x00\x88\x01\x01\x42\x11\n\x0f_midpoint_index\"N\n\tSuccessor\x12\x12\n\x05swhid\x18\x01 \x01(\tH\x00\x88\x01\x01\x12#\n\x05label\x18\x02 \x03(\x0b\x32\x14.swh.graph.EdgeLabelB\x08\n\x06_swhid\"U\n\x0b\x43ontentData\x12\x13\n\x06length\x18\x01 \x01(\x03H\x00\x88\x01\x01\x12\x17\n\nis_skipped\x18\x02 \x01(\x08H\x01\x88\x01\x01\x42\t\n\x07_lengthB\r\n\x0b_is_skipped\"\xc6\x02\n\x0cRevisionData\x12\x13\n\x06\x61uthor\x18\x01 \x01(\x03H\x00\x88\x01\x01\x12\x18\n\x0b\x61uthor_date\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1f\n\x12\x61uthor_date_offset\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x16\n\tcommitter\x18\x04 \x01(\x03H\x03\x88\x01\x01\x12\x1b\n\x0e\x63ommitter_date\x18\x05 \x01(\x03H\x04\x88\x01\x01\x12\"\n\x15\x63ommitter_date_offset\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x14\n\x07message\x18\x07 \x01(\x0cH\x06\x88\x01\x01\x42\t\n\x07_authorB\x0e\n\x0c_author_dateB\x15\n\x13_author_date_offsetB\x0c\n\n_committerB\x11\n\x0f_committer_dateB\x18\n\x16_committer_date_offsetB\n\n\x08_message\"\xcd\x01\n\x0bReleaseData\x12\x13\n\x06\x61uthor\x18\x01 \x01(\x03H\x00\x88\x01\x01\x12\x18\n\x0b\x61uthor_date\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1f\n\x12\x61uthor_date_offset\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x11\n\x04name\x18\x04 \x01(\x0cH\x03\x88\x01\x01\x12\x14\n\x07message\x18\x05 \x01(\x0cH\x04\x88\x01\x01\x42\t\n\x07_authorB\x0e\n\x0c_author_dateB\x15\n\x13_author_date_offsetB\x07\n\x05_nameB\n\n\x08_message\"&\n\nOriginData\x12\x10\n\x03url\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x06\n\x04_url\"-\n\tEdgeLabel\x12\x0c\n\x04name\x18\x01 \x01(\x0c\x12\x12\n\npermission\x18\x02 \x01(\x05\"\x1e\n\rCountResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\"\x0e\n\x0cStatsRequest\"\x9b\x02\n\rStatsResponse\x12\x11\n\tnum_nodes\x18\x01 \x01(\x03\x12\x11\n\tnum_edges\x18\x02 \x01(\x03\x12\x19\n\x11\x63ompression_ratio\x18\x03 \x01(\x01\x12\x15\n\rbits_per_node\x18\x04 \x01(\x01\x12\x15\n\rbits_per_edge\x18\x05 \x01(\x01\x12\x14\n\x0c\x61vg_locality\x18\x06 \x01(\x01\x12\x14\n\x0cindegree_min\x18\x07 \x01(\x03\x12\x14\n\x0cindegree_max\x18\x08 \x01(\x03\x12\x14\n\x0cindegree_avg\x18\t \x01(\x01\x12\x15\n\routdegree_min\x18\n \x01(\x03\x12\x15\n\routdegree_max\x18\x0b \x01(\x03\x12\x15\n\routdegree_avg\x18\x0c \x01(\x01*+\n\x0eGraphDirection\x12\x0b\n\x07\x46ORWARD\x10\x00\x12\x0c\n\x08\x42\x41\x43KWARD\x10\x01\x32\xcf\x03\n\x10TraversalService\x12\x35\n\x07GetNode\x12\x19.swh.graph.GetNodeRequest\x1a\x0f.swh.graph.Node\x12:\n\x08Traverse\x12\x1b.swh.graph.TraversalRequest\x1a\x0f.swh.graph.Node0\x01\x12;\n\nFindPathTo\x12\x1c.swh.graph.FindPathToRequest\x1a\x0f.swh.graph.Path\x12\x45\n\x0f\x46indPathBetween\x12!.swh.graph.FindPathBetweenRequest\x1a\x0f.swh.graph.Path\x12\x43\n\nCountNodes\x12\x1b.swh.graph.TraversalRequest\x1a\x18.swh.graph.CountResponse\x12\x43\n\nCountEdges\x12\x1b.swh.graph.TraversalRequest\x1a\x18.swh.graph.CountResponse\x12:\n\x05Stats\x12\x17.swh.graph.StatsRequest\x1a\x18.swh.graph.StatsResponseB0\n\x1eorg.softwareheritage.graph.rpcB\x0cGraphServiceP\x01\x62\x06proto3') + +_GRAPHDIRECTION = DESCRIPTOR.enum_types_by_name['GraphDirection'] +GraphDirection = enum_type_wrapper.EnumTypeWrapper(_GRAPHDIRECTION) +FORWARD = 0 +BACKWARD = 1 + + +_GETNODEREQUEST = DESCRIPTOR.message_types_by_name['GetNodeRequest'] +_TRAVERSALREQUEST = DESCRIPTOR.message_types_by_name['TraversalRequest'] +_FINDPATHTOREQUEST = DESCRIPTOR.message_types_by_name['FindPathToRequest'] +_FINDPATHBETWEENREQUEST = DESCRIPTOR.message_types_by_name['FindPathBetweenRequest'] +_NODEFILTER = DESCRIPTOR.message_types_by_name['NodeFilter'] +_NODE = DESCRIPTOR.message_types_by_name['Node'] +_PATH = DESCRIPTOR.message_types_by_name['Path'] +_SUCCESSOR = DESCRIPTOR.message_types_by_name['Successor'] +_CONTENTDATA = DESCRIPTOR.message_types_by_name['ContentData'] +_REVISIONDATA = DESCRIPTOR.message_types_by_name['RevisionData'] +_RELEASEDATA = DESCRIPTOR.message_types_by_name['ReleaseData'] +_ORIGINDATA = DESCRIPTOR.message_types_by_name['OriginData'] +_EDGELABEL = DESCRIPTOR.message_types_by_name['EdgeLabel'] +_COUNTRESPONSE = DESCRIPTOR.message_types_by_name['CountResponse'] +_STATSREQUEST = DESCRIPTOR.message_types_by_name['StatsRequest'] +_STATSRESPONSE = DESCRIPTOR.message_types_by_name['StatsResponse'] +GetNodeRequest = _reflection.GeneratedProtocolMessageType('GetNodeRequest', (_message.Message,), { + 'DESCRIPTOR' : _GETNODEREQUEST, + '__module__' : 'swh.graph.grpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.GetNodeRequest) + }) +_sym_db.RegisterMessage(GetNodeRequest) + +TraversalRequest = _reflection.GeneratedProtocolMessageType('TraversalRequest', (_message.Message,), { + 'DESCRIPTOR' : _TRAVERSALREQUEST, + '__module__' : 'swh.graph.grpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.TraversalRequest) + }) +_sym_db.RegisterMessage(TraversalRequest) + +FindPathToRequest = _reflection.GeneratedProtocolMessageType('FindPathToRequest', (_message.Message,), { + 'DESCRIPTOR' : _FINDPATHTOREQUEST, + '__module__' : 'swh.graph.grpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.FindPathToRequest) + }) +_sym_db.RegisterMessage(FindPathToRequest) + +FindPathBetweenRequest = _reflection.GeneratedProtocolMessageType('FindPathBetweenRequest', (_message.Message,), { + 'DESCRIPTOR' : _FINDPATHBETWEENREQUEST, + '__module__' : 'swh.graph.grpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.FindPathBetweenRequest) + }) +_sym_db.RegisterMessage(FindPathBetweenRequest) + +NodeFilter = _reflection.GeneratedProtocolMessageType('NodeFilter', (_message.Message,), { + 'DESCRIPTOR' : _NODEFILTER, + '__module__' : 'swh.graph.grpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.NodeFilter) + }) +_sym_db.RegisterMessage(NodeFilter) + +Node = _reflection.GeneratedProtocolMessageType('Node', (_message.Message,), { + 'DESCRIPTOR' : _NODE, + '__module__' : 'swh.graph.grpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.Node) + }) +_sym_db.RegisterMessage(Node) + +Path = _reflection.GeneratedProtocolMessageType('Path', (_message.Message,), { + 'DESCRIPTOR' : _PATH, + '__module__' : 'swh.graph.grpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.Path) + }) +_sym_db.RegisterMessage(Path) + +Successor = _reflection.GeneratedProtocolMessageType('Successor', (_message.Message,), { + 'DESCRIPTOR' : _SUCCESSOR, + '__module__' : 'swh.graph.grpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.Successor) + }) +_sym_db.RegisterMessage(Successor) + +ContentData = _reflection.GeneratedProtocolMessageType('ContentData', (_message.Message,), { + 'DESCRIPTOR' : _CONTENTDATA, + '__module__' : 'swh.graph.grpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.ContentData) + }) +_sym_db.RegisterMessage(ContentData) + +RevisionData = _reflection.GeneratedProtocolMessageType('RevisionData', (_message.Message,), { + 'DESCRIPTOR' : _REVISIONDATA, + '__module__' : 'swh.graph.grpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.RevisionData) + }) +_sym_db.RegisterMessage(RevisionData) + +ReleaseData = _reflection.GeneratedProtocolMessageType('ReleaseData', (_message.Message,), { + 'DESCRIPTOR' : _RELEASEDATA, + '__module__' : 'swh.graph.grpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.ReleaseData) + }) +_sym_db.RegisterMessage(ReleaseData) + +OriginData = _reflection.GeneratedProtocolMessageType('OriginData', (_message.Message,), { + 'DESCRIPTOR' : _ORIGINDATA, + '__module__' : 'swh.graph.grpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.OriginData) + }) +_sym_db.RegisterMessage(OriginData) + +EdgeLabel = _reflection.GeneratedProtocolMessageType('EdgeLabel', (_message.Message,), { + 'DESCRIPTOR' : _EDGELABEL, + '__module__' : 'swh.graph.grpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.EdgeLabel) + }) +_sym_db.RegisterMessage(EdgeLabel) + +CountResponse = _reflection.GeneratedProtocolMessageType('CountResponse', (_message.Message,), { + 'DESCRIPTOR' : _COUNTRESPONSE, + '__module__' : 'swh.graph.grpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.CountResponse) + }) +_sym_db.RegisterMessage(CountResponse) + +StatsRequest = _reflection.GeneratedProtocolMessageType('StatsRequest', (_message.Message,), { + 'DESCRIPTOR' : _STATSREQUEST, + '__module__' : 'swh.graph.grpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.StatsRequest) + }) +_sym_db.RegisterMessage(StatsRequest) + +StatsResponse = _reflection.GeneratedProtocolMessageType('StatsResponse', (_message.Message,), { + 'DESCRIPTOR' : _STATSRESPONSE, + '__module__' : 'swh.graph.grpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.StatsResponse) + }) +_sym_db.RegisterMessage(StatsResponse) + +_TRAVERSALSERVICE = DESCRIPTOR.services_by_name['TraversalService'] +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\036org.softwareheritage.graph.rpcB\014GraphServiceP\001' + _GRAPHDIRECTION._serialized_start=2854 + _GRAPHDIRECTION._serialized_end=2897 + _GETNODEREQUEST._serialized_start=78 + _GETNODEREQUEST._serialized_end=165 + _TRAVERSALREQUEST._serialized_start=168 + _TRAVERSALREQUEST._serialized_end=512 + _FINDPATHTOREQUEST._serialized_start=515 + _FINDPATHTOREQUEST._serialized_end=794 + _FINDPATHBETWEENREQUEST._serialized_start=797 + _FINDPATHBETWEENREQUEST._serialized_end=1182 + _NODEFILTER._serialized_start=1185 + _NODEFILTER._serialized_end=1363 + _NODE._serialized_start=1366 + _NODE._serialized_end=1640 + _PATH._serialized_start=1642 + _PATH._serialized_end=1727 + _SUCCESSOR._serialized_start=1729 + _SUCCESSOR._serialized_end=1807 + _CONTENTDATA._serialized_start=1809 + _CONTENTDATA._serialized_end=1894 + _REVISIONDATA._serialized_start=1897 + _REVISIONDATA._serialized_end=2223 + _RELEASEDATA._serialized_start=2226 + _RELEASEDATA._serialized_end=2431 + _ORIGINDATA._serialized_start=2433 + _ORIGINDATA._serialized_end=2471 + _EDGELABEL._serialized_start=2473 + _EDGELABEL._serialized_end=2518 + _COUNTRESPONSE._serialized_start=2520 + _COUNTRESPONSE._serialized_end=2550 + _STATSREQUEST._serialized_start=2552 + _STATSREQUEST._serialized_end=2566 + _STATSRESPONSE._serialized_start=2569 + _STATSRESPONSE._serialized_end=2852 + _TRAVERSALSERVICE._serialized_start=2900 + _TRAVERSALSERVICE._serialized_end=3363 +# @@protoc_insertion_point(module_scope) diff --git a/swh/graph/rpc/swhgraph_pb2.pyi b/swh/graph/grpc/swhgraph_pb2.pyi similarity index 60% rename from swh/graph/rpc/swhgraph_pb2.pyi rename to swh/graph/grpc/swhgraph_pb2.pyi index 1d099bf..8e108b0 100644 --- a/swh/graph/rpc/swhgraph_pb2.pyi +++ b/swh/graph/grpc/swhgraph_pb2.pyi @@ -1,685 +1,683 @@ """ @generated by mypy-protobuf. Do not edit manually! isort:skip_file """ import builtins +import collections.abc import google.protobuf.descriptor import google.protobuf.field_mask_pb2 import google.protobuf.internal.containers import google.protobuf.internal.enum_type_wrapper import google.protobuf.message +import sys import typing -import typing_extensions + +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions DESCRIPTOR: google.protobuf.descriptor.FileDescriptor class _GraphDirection: - ValueType = typing.NewType('ValueType', builtins.int) + ValueType = typing.NewType("ValueType", builtins.int) V: typing_extensions.TypeAlias = ValueType -class _GraphDirectionEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_GraphDirection.ValueType], builtins.type): + +class _GraphDirectionEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_GraphDirection.ValueType], builtins.type): # noqa: F821 DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor FORWARD: _GraphDirection.ValueType # 0 """Forward DAG: ori -> snp -> rel -> rev -> dir -> cnt""" - BACKWARD: _GraphDirection.ValueType # 1 """Transposed DAG: cnt -> dir -> rev -> rel -> snp -> ori""" class GraphDirection(_GraphDirection, metaclass=_GraphDirectionEnumTypeWrapper): """Direction of the graph""" - pass FORWARD: GraphDirection.ValueType # 0 """Forward DAG: ori -> snp -> rel -> rev -> dir -> cnt""" - BACKWARD: GraphDirection.ValueType # 1 """Transposed DAG: cnt -> dir -> rev -> rel -> snp -> ori""" - global___GraphDirection = GraphDirection - class GetNodeRequest(google.protobuf.message.Message): """Describe a node to return""" + DESCRIPTOR: google.protobuf.descriptor.Descriptor + SWHID_FIELD_NUMBER: builtins.int MASK_FIELD_NUMBER: builtins.int - swhid: typing.Text + swhid: builtins.str """SWHID of the node to return""" - @property def mask(self) -> google.protobuf.field_mask_pb2.FieldMask: """FieldMask of which fields are to be returned (e.g., "swhid,cnt.length"). By default, all fields are returned. """ - pass - def __init__(self, + def __init__( + self, *, - swhid: typing.Text = ..., - mask: typing.Optional[google.protobuf.field_mask_pb2.FieldMask] = ..., - ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_mask",b"_mask","mask",b"mask"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_mask",b"_mask","mask",b"mask","swhid",b"swhid"]) -> None: ... - def WhichOneof(self, oneof_group: typing_extensions.Literal["_mask",b"_mask"]) -> typing.Optional[typing_extensions.Literal["mask"]]: ... + swhid: builtins.str = ..., + mask: google.protobuf.field_mask_pb2.FieldMask | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_mask", b"_mask", "mask", b"mask"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_mask", b"_mask", "mask", b"mask", "swhid", b"swhid"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_mask", b"_mask"]) -> typing_extensions.Literal["mask"] | None: ... + global___GetNodeRequest = GetNodeRequest class TraversalRequest(google.protobuf.message.Message): """TraversalRequest describes how a breadth-first traversal should be performed, and what should be returned to the client. """ + DESCRIPTOR: google.protobuf.descriptor.Descriptor + SRC_FIELD_NUMBER: builtins.int DIRECTION_FIELD_NUMBER: builtins.int EDGES_FIELD_NUMBER: builtins.int MAX_EDGES_FIELD_NUMBER: builtins.int MIN_DEPTH_FIELD_NUMBER: builtins.int MAX_DEPTH_FIELD_NUMBER: builtins.int RETURN_NODES_FIELD_NUMBER: builtins.int MASK_FIELD_NUMBER: builtins.int @property - def src(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[typing.Text]: + def src(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: """Set of source nodes (SWHIDs)""" - pass direction: global___GraphDirection.ValueType """Direction of the graph to traverse. Defaults to FORWARD.""" - - edges: typing.Text + edges: builtins.str """Edge restriction string (e.g. "rev:dir,dir:cnt"). Defaults to "*" (all). """ - max_edges: builtins.int """Maximum number of edges accessed in the traversal, after which it stops. Defaults to infinite. """ - min_depth: builtins.int """Do not return nodes with a depth lower than this number. By default, all depths are returned. """ - max_depth: builtins.int """Maximum depth of the traversal, after which it stops. Defaults to infinite. """ - @property def return_nodes(self) -> global___NodeFilter: """Filter which nodes will be sent to the stream. By default, all nodes are returned. """ - pass @property def mask(self) -> google.protobuf.field_mask_pb2.FieldMask: """FieldMask of which fields are to be returned (e.g., "swhid,cnt.length"). By default, all fields are returned. """ - pass - def __init__(self, + def __init__( + self, *, - src: typing.Optional[typing.Iterable[typing.Text]] = ..., + src: collections.abc.Iterable[builtins.str] | None = ..., direction: global___GraphDirection.ValueType = ..., - edges: typing.Optional[typing.Text] = ..., - max_edges: typing.Optional[builtins.int] = ..., - min_depth: typing.Optional[builtins.int] = ..., - max_depth: typing.Optional[builtins.int] = ..., - return_nodes: typing.Optional[global___NodeFilter] = ..., - mask: typing.Optional[google.protobuf.field_mask_pb2.FieldMask] = ..., - ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_edges",b"_edges","_mask",b"_mask","_max_depth",b"_max_depth","_max_edges",b"_max_edges","_min_depth",b"_min_depth","_return_nodes",b"_return_nodes","edges",b"edges","mask",b"mask","max_depth",b"max_depth","max_edges",b"max_edges","min_depth",b"min_depth","return_nodes",b"return_nodes"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_edges",b"_edges","_mask",b"_mask","_max_depth",b"_max_depth","_max_edges",b"_max_edges","_min_depth",b"_min_depth","_return_nodes",b"_return_nodes","direction",b"direction","edges",b"edges","mask",b"mask","max_depth",b"max_depth","max_edges",b"max_edges","min_depth",b"min_depth","return_nodes",b"return_nodes","src",b"src"]) -> None: ... + edges: builtins.str | None = ..., + max_edges: builtins.int | None = ..., + min_depth: builtins.int | None = ..., + max_depth: builtins.int | None = ..., + return_nodes: global___NodeFilter | None = ..., + mask: google.protobuf.field_mask_pb2.FieldMask | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_edges", b"_edges", "_mask", b"_mask", "_max_depth", b"_max_depth", "_max_edges", b"_max_edges", "_min_depth", b"_min_depth", "_return_nodes", b"_return_nodes", "edges", b"edges", "mask", b"mask", "max_depth", b"max_depth", "max_edges", b"max_edges", "min_depth", b"min_depth", "return_nodes", b"return_nodes"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_edges", b"_edges", "_mask", b"_mask", "_max_depth", b"_max_depth", "_max_edges", b"_max_edges", "_min_depth", b"_min_depth", "_return_nodes", b"_return_nodes", "direction", b"direction", "edges", b"edges", "mask", b"mask", "max_depth", b"max_depth", "max_edges", b"max_edges", "min_depth", b"min_depth", "return_nodes", b"return_nodes", "src", b"src"]) -> None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_edges",b"_edges"]) -> typing.Optional[typing_extensions.Literal["edges"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_edges", b"_edges"]) -> typing_extensions.Literal["edges"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_mask",b"_mask"]) -> typing.Optional[typing_extensions.Literal["mask"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_mask", b"_mask"]) -> typing_extensions.Literal["mask"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_depth",b"_max_depth"]) -> typing.Optional[typing_extensions.Literal["max_depth"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_depth", b"_max_depth"]) -> typing_extensions.Literal["max_depth"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_edges",b"_max_edges"]) -> typing.Optional[typing_extensions.Literal["max_edges"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_edges", b"_max_edges"]) -> typing_extensions.Literal["max_edges"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_min_depth",b"_min_depth"]) -> typing.Optional[typing_extensions.Literal["min_depth"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_min_depth", b"_min_depth"]) -> typing_extensions.Literal["min_depth"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_return_nodes",b"_return_nodes"]) -> typing.Optional[typing_extensions.Literal["return_nodes"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_return_nodes", b"_return_nodes"]) -> typing_extensions.Literal["return_nodes"] | None: ... + global___TraversalRequest = TraversalRequest class FindPathToRequest(google.protobuf.message.Message): """FindPathToRequest describes a request to find a shortest path between a set of nodes and a given target criteria, as well as what should be returned in the path. """ + DESCRIPTOR: google.protobuf.descriptor.Descriptor + SRC_FIELD_NUMBER: builtins.int TARGET_FIELD_NUMBER: builtins.int DIRECTION_FIELD_NUMBER: builtins.int EDGES_FIELD_NUMBER: builtins.int MAX_EDGES_FIELD_NUMBER: builtins.int MAX_DEPTH_FIELD_NUMBER: builtins.int MASK_FIELD_NUMBER: builtins.int @property - def src(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[typing.Text]: + def src(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: """Set of source nodes (SWHIDs)""" - pass @property def target(self) -> global___NodeFilter: """Target criteria, i.e., what constitutes a valid path destination.""" - pass direction: global___GraphDirection.ValueType """Direction of the graph to traverse. Defaults to FORWARD.""" - - edges: typing.Text + edges: builtins.str """Edge restriction string (e.g. "rev:dir,dir:cnt"). Defaults to "*" (all). """ - max_edges: builtins.int """Maximum number of edges accessed in the traversal, after which it stops. Defaults to infinite. """ - max_depth: builtins.int """Maximum depth of the traversal, after which it stops. Defaults to infinite. """ - @property def mask(self) -> google.protobuf.field_mask_pb2.FieldMask: """FieldMask of which fields are to be returned (e.g., "swhid,cnt.length"). By default, all fields are returned. """ - pass - def __init__(self, + def __init__( + self, *, - src: typing.Optional[typing.Iterable[typing.Text]] = ..., - target: typing.Optional[global___NodeFilter] = ..., + src: collections.abc.Iterable[builtins.str] | None = ..., + target: global___NodeFilter | None = ..., direction: global___GraphDirection.ValueType = ..., - edges: typing.Optional[typing.Text] = ..., - max_edges: typing.Optional[builtins.int] = ..., - max_depth: typing.Optional[builtins.int] = ..., - mask: typing.Optional[google.protobuf.field_mask_pb2.FieldMask] = ..., - ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_edges",b"_edges","_mask",b"_mask","_max_depth",b"_max_depth","_max_edges",b"_max_edges","edges",b"edges","mask",b"mask","max_depth",b"max_depth","max_edges",b"max_edges","target",b"target"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_edges",b"_edges","_mask",b"_mask","_max_depth",b"_max_depth","_max_edges",b"_max_edges","direction",b"direction","edges",b"edges","mask",b"mask","max_depth",b"max_depth","max_edges",b"max_edges","src",b"src","target",b"target"]) -> None: ... + edges: builtins.str | None = ..., + max_edges: builtins.int | None = ..., + max_depth: builtins.int | None = ..., + mask: google.protobuf.field_mask_pb2.FieldMask | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_edges", b"_edges", "_mask", b"_mask", "_max_depth", b"_max_depth", "_max_edges", b"_max_edges", "edges", b"edges", "mask", b"mask", "max_depth", b"max_depth", "max_edges", b"max_edges", "target", b"target"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_edges", b"_edges", "_mask", b"_mask", "_max_depth", b"_max_depth", "_max_edges", b"_max_edges", "direction", b"direction", "edges", b"edges", "mask", b"mask", "max_depth", b"max_depth", "max_edges", b"max_edges", "src", b"src", "target", b"target"]) -> None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_edges",b"_edges"]) -> typing.Optional[typing_extensions.Literal["edges"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_edges", b"_edges"]) -> typing_extensions.Literal["edges"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_mask",b"_mask"]) -> typing.Optional[typing_extensions.Literal["mask"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_mask", b"_mask"]) -> typing_extensions.Literal["mask"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_depth",b"_max_depth"]) -> typing.Optional[typing_extensions.Literal["max_depth"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_depth", b"_max_depth"]) -> typing_extensions.Literal["max_depth"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_edges",b"_max_edges"]) -> typing.Optional[typing_extensions.Literal["max_edges"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_edges", b"_max_edges"]) -> typing_extensions.Literal["max_edges"] | None: ... + global___FindPathToRequest = FindPathToRequest class FindPathBetweenRequest(google.protobuf.message.Message): """FindPathToRequest describes a request to find a shortest path between a set of source nodes and a set of destination nodes. It works by performing a bidirectional breadth-first traversal from both sets at the same time. """ + DESCRIPTOR: google.protobuf.descriptor.Descriptor + SRC_FIELD_NUMBER: builtins.int DST_FIELD_NUMBER: builtins.int DIRECTION_FIELD_NUMBER: builtins.int DIRECTION_REVERSE_FIELD_NUMBER: builtins.int EDGES_FIELD_NUMBER: builtins.int EDGES_REVERSE_FIELD_NUMBER: builtins.int MAX_EDGES_FIELD_NUMBER: builtins.int MAX_DEPTH_FIELD_NUMBER: builtins.int MASK_FIELD_NUMBER: builtins.int @property - def src(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[typing.Text]: + def src(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: """Set of source nodes (SWHIDs)""" - pass @property - def dst(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[typing.Text]: + def dst(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: """Set of destination nodes (SWHIDs)""" - pass direction: global___GraphDirection.ValueType """Direction of the graph to traverse from the source set. Defaults to FORWARD. """ - direction_reverse: global___GraphDirection.ValueType """Direction of the graph to traverse from the destination set. Defaults to the opposite of `direction`. If direction and direction_reverse are identical, it will find the first common successor of both sets in the given direction. """ - - edges: typing.Text + edges: builtins.str """Edge restriction string for the traversal from the source set. (e.g. "rev:dir,dir:cnt"). Defaults to "*" (all). """ - - edges_reverse: typing.Text + edges_reverse: builtins.str """Edge restriction string for the reverse traversal from the destination set. If not specified: - If `edges` is not specified either, defaults to "*" - If direction == direction_reverse, defaults to `edges` - If direction != direction_reverse, defaults to the reverse of `edges` (e.g. "rev:dir" becomes "dir:rev"). """ - max_edges: builtins.int """Maximum number of edges accessed in the traversal, after which it stops. Defaults to infinite. """ - max_depth: builtins.int """Maximum depth of the traversal, after which it stops. Defaults to infinite. """ - @property def mask(self) -> google.protobuf.field_mask_pb2.FieldMask: """FieldMask of which fields are to be returned (e.g., "swhid,cnt.length"). By default, all fields are returned. """ - pass - def __init__(self, + def __init__( + self, *, - src: typing.Optional[typing.Iterable[typing.Text]] = ..., - dst: typing.Optional[typing.Iterable[typing.Text]] = ..., + src: collections.abc.Iterable[builtins.str] | None = ..., + dst: collections.abc.Iterable[builtins.str] | None = ..., direction: global___GraphDirection.ValueType = ..., - direction_reverse: typing.Optional[global___GraphDirection.ValueType] = ..., - edges: typing.Optional[typing.Text] = ..., - edges_reverse: typing.Optional[typing.Text] = ..., - max_edges: typing.Optional[builtins.int] = ..., - max_depth: typing.Optional[builtins.int] = ..., - mask: typing.Optional[google.protobuf.field_mask_pb2.FieldMask] = ..., - ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_direction_reverse",b"_direction_reverse","_edges",b"_edges","_edges_reverse",b"_edges_reverse","_mask",b"_mask","_max_depth",b"_max_depth","_max_edges",b"_max_edges","direction_reverse",b"direction_reverse","edges",b"edges","edges_reverse",b"edges_reverse","mask",b"mask","max_depth",b"max_depth","max_edges",b"max_edges"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_direction_reverse",b"_direction_reverse","_edges",b"_edges","_edges_reverse",b"_edges_reverse","_mask",b"_mask","_max_depth",b"_max_depth","_max_edges",b"_max_edges","direction",b"direction","direction_reverse",b"direction_reverse","dst",b"dst","edges",b"edges","edges_reverse",b"edges_reverse","mask",b"mask","max_depth",b"max_depth","max_edges",b"max_edges","src",b"src"]) -> None: ... + direction_reverse: global___GraphDirection.ValueType | None = ..., + edges: builtins.str | None = ..., + edges_reverse: builtins.str | None = ..., + max_edges: builtins.int | None = ..., + max_depth: builtins.int | None = ..., + mask: google.protobuf.field_mask_pb2.FieldMask | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_direction_reverse", b"_direction_reverse", "_edges", b"_edges", "_edges_reverse", b"_edges_reverse", "_mask", b"_mask", "_max_depth", b"_max_depth", "_max_edges", b"_max_edges", "direction_reverse", b"direction_reverse", "edges", b"edges", "edges_reverse", b"edges_reverse", "mask", b"mask", "max_depth", b"max_depth", "max_edges", b"max_edges"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_direction_reverse", b"_direction_reverse", "_edges", b"_edges", "_edges_reverse", b"_edges_reverse", "_mask", b"_mask", "_max_depth", b"_max_depth", "_max_edges", b"_max_edges", "direction", b"direction", "direction_reverse", b"direction_reverse", "dst", b"dst", "edges", b"edges", "edges_reverse", b"edges_reverse", "mask", b"mask", "max_depth", b"max_depth", "max_edges", b"max_edges", "src", b"src"]) -> None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_direction_reverse",b"_direction_reverse"]) -> typing.Optional[typing_extensions.Literal["direction_reverse"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_direction_reverse", b"_direction_reverse"]) -> typing_extensions.Literal["direction_reverse"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_edges",b"_edges"]) -> typing.Optional[typing_extensions.Literal["edges"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_edges", b"_edges"]) -> typing_extensions.Literal["edges"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_edges_reverse",b"_edges_reverse"]) -> typing.Optional[typing_extensions.Literal["edges_reverse"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_edges_reverse", b"_edges_reverse"]) -> typing_extensions.Literal["edges_reverse"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_mask",b"_mask"]) -> typing.Optional[typing_extensions.Literal["mask"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_mask", b"_mask"]) -> typing_extensions.Literal["mask"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_depth",b"_max_depth"]) -> typing.Optional[typing_extensions.Literal["max_depth"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_depth", b"_max_depth"]) -> typing_extensions.Literal["max_depth"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_edges",b"_max_edges"]) -> typing.Optional[typing_extensions.Literal["max_edges"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_edges", b"_max_edges"]) -> typing_extensions.Literal["max_edges"] | None: ... + global___FindPathBetweenRequest = FindPathBetweenRequest class NodeFilter(google.protobuf.message.Message): """Represents various criteria that make a given node "valid". A node is only valid if all the subcriteria present in this message are fulfilled. """ + DESCRIPTOR: google.protobuf.descriptor.Descriptor + TYPES_FIELD_NUMBER: builtins.int MIN_TRAVERSAL_SUCCESSORS_FIELD_NUMBER: builtins.int MAX_TRAVERSAL_SUCCESSORS_FIELD_NUMBER: builtins.int - types: typing.Text + types: builtins.str """Node restriction string. (e.g. "dir,cnt,rev"). Defaults to "*" (all).""" - min_traversal_successors: builtins.int """Minimum number of successors encountered *during the traversal*. Default: no constraint """ - max_traversal_successors: builtins.int """Maximum number of successors encountered *during the traversal*. Default: no constraint """ - - def __init__(self, + def __init__( + self, *, - types: typing.Optional[typing.Text] = ..., - min_traversal_successors: typing.Optional[builtins.int] = ..., - max_traversal_successors: typing.Optional[builtins.int] = ..., - ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_max_traversal_successors",b"_max_traversal_successors","_min_traversal_successors",b"_min_traversal_successors","_types",b"_types","max_traversal_successors",b"max_traversal_successors","min_traversal_successors",b"min_traversal_successors","types",b"types"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_max_traversal_successors",b"_max_traversal_successors","_min_traversal_successors",b"_min_traversal_successors","_types",b"_types","max_traversal_successors",b"max_traversal_successors","min_traversal_successors",b"min_traversal_successors","types",b"types"]) -> None: ... + types: builtins.str | None = ..., + min_traversal_successors: builtins.int | None = ..., + max_traversal_successors: builtins.int | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_max_traversal_successors", b"_max_traversal_successors", "_min_traversal_successors", b"_min_traversal_successors", "_types", b"_types", "max_traversal_successors", b"max_traversal_successors", "min_traversal_successors", b"min_traversal_successors", "types", b"types"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_max_traversal_successors", b"_max_traversal_successors", "_min_traversal_successors", b"_min_traversal_successors", "_types", b"_types", "max_traversal_successors", b"max_traversal_successors", "min_traversal_successors", b"min_traversal_successors", "types", b"types"]) -> None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_traversal_successors",b"_max_traversal_successors"]) -> typing.Optional[typing_extensions.Literal["max_traversal_successors"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_traversal_successors", b"_max_traversal_successors"]) -> typing_extensions.Literal["max_traversal_successors"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_min_traversal_successors",b"_min_traversal_successors"]) -> typing.Optional[typing_extensions.Literal["min_traversal_successors"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_min_traversal_successors", b"_min_traversal_successors"]) -> typing_extensions.Literal["min_traversal_successors"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_types",b"_types"]) -> typing.Optional[typing_extensions.Literal["types"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_types", b"_types"]) -> typing_extensions.Literal["types"] | None: ... + global___NodeFilter = NodeFilter class Node(google.protobuf.message.Message): """Represents a node in the graph.""" + DESCRIPTOR: google.protobuf.descriptor.Descriptor + SWHID_FIELD_NUMBER: builtins.int SUCCESSOR_FIELD_NUMBER: builtins.int NUM_SUCCESSORS_FIELD_NUMBER: builtins.int CNT_FIELD_NUMBER: builtins.int REV_FIELD_NUMBER: builtins.int REL_FIELD_NUMBER: builtins.int ORI_FIELD_NUMBER: builtins.int - swhid: typing.Text + swhid: builtins.str """The SWHID of the graph node.""" - @property def successor(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Successor]: """List of relevant successors of this node.""" - pass num_successors: builtins.int """Number of relevant successors.""" - @property def cnt(self) -> global___ContentData: ... @property def rev(self) -> global___RevisionData: ... @property def rel(self) -> global___ReleaseData: ... @property def ori(self) -> global___OriginData: ... - def __init__(self, + def __init__( + self, *, - swhid: typing.Text = ..., - successor: typing.Optional[typing.Iterable[global___Successor]] = ..., - num_successors: typing.Optional[builtins.int] = ..., - cnt: typing.Optional[global___ContentData] = ..., - rev: typing.Optional[global___RevisionData] = ..., - rel: typing.Optional[global___ReleaseData] = ..., - ori: typing.Optional[global___OriginData] = ..., - ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_num_successors",b"_num_successors","cnt",b"cnt","data",b"data","num_successors",b"num_successors","ori",b"ori","rel",b"rel","rev",b"rev"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_num_successors",b"_num_successors","cnt",b"cnt","data",b"data","num_successors",b"num_successors","ori",b"ori","rel",b"rel","rev",b"rev","successor",b"successor","swhid",b"swhid"]) -> None: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_num_successors",b"_num_successors"]) -> typing.Optional[typing_extensions.Literal["num_successors"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["data",b"data"]) -> typing.Optional[typing_extensions.Literal["cnt","rev","rel","ori"]]: ... + swhid: builtins.str = ..., + successor: collections.abc.Iterable[global___Successor] | None = ..., + num_successors: builtins.int | None = ..., + cnt: global___ContentData | None = ..., + rev: global___RevisionData | None = ..., + rel: global___ReleaseData | None = ..., + ori: global___OriginData | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_num_successors", b"_num_successors", "cnt", b"cnt", "data", b"data", "num_successors", b"num_successors", "ori", b"ori", "rel", b"rel", "rev", b"rev"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_num_successors", b"_num_successors", "cnt", b"cnt", "data", b"data", "num_successors", b"num_successors", "ori", b"ori", "rel", b"rel", "rev", b"rev", "successor", b"successor", "swhid", b"swhid"]) -> None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_num_successors", b"_num_successors"]) -> typing_extensions.Literal["num_successors"] | None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["data", b"data"]) -> typing_extensions.Literal["cnt", "rev", "rel", "ori"] | None: ... + global___Node = Node class Path(google.protobuf.message.Message): """Represents a path in the graph.""" + DESCRIPTOR: google.protobuf.descriptor.Descriptor + NODE_FIELD_NUMBER: builtins.int MIDPOINT_INDEX_FIELD_NUMBER: builtins.int @property def node(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Node]: """List of nodes in the path, from source to destination""" - pass midpoint_index: builtins.int """Index of the "midpoint" of the path. For paths obtained with bidirectional search queries, this is the node that joined the two sets together. When looking for a common ancestor between two nodes by performing a FindPathBetween search with two backward graphs, this will be the index of the common ancestor in the path. """ - - def __init__(self, + def __init__( + self, *, - node: typing.Optional[typing.Iterable[global___Node]] = ..., - midpoint_index: typing.Optional[builtins.int] = ..., - ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_midpoint_index",b"_midpoint_index","midpoint_index",b"midpoint_index"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_midpoint_index",b"_midpoint_index","midpoint_index",b"midpoint_index","node",b"node"]) -> None: ... - def WhichOneof(self, oneof_group: typing_extensions.Literal["_midpoint_index",b"_midpoint_index"]) -> typing.Optional[typing_extensions.Literal["midpoint_index"]]: ... + node: collections.abc.Iterable[global___Node] | None = ..., + midpoint_index: builtins.int | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_midpoint_index", b"_midpoint_index", "midpoint_index", b"midpoint_index"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_midpoint_index", b"_midpoint_index", "midpoint_index", b"midpoint_index", "node", b"node"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_midpoint_index", b"_midpoint_index"]) -> typing_extensions.Literal["midpoint_index"] | None: ... + global___Path = Path class Successor(google.protobuf.message.Message): """Represents a successor of a given node.""" + DESCRIPTOR: google.protobuf.descriptor.Descriptor + SWHID_FIELD_NUMBER: builtins.int LABEL_FIELD_NUMBER: builtins.int - swhid: typing.Text + swhid: builtins.str """The SWHID of the successor""" - @property def label(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___EdgeLabel]: """A list of edge labels for the given edge""" - pass - def __init__(self, + def __init__( + self, *, - swhid: typing.Optional[typing.Text] = ..., - label: typing.Optional[typing.Iterable[global___EdgeLabel]] = ..., - ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_swhid",b"_swhid","swhid",b"swhid"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_swhid",b"_swhid","label",b"label","swhid",b"swhid"]) -> None: ... - def WhichOneof(self, oneof_group: typing_extensions.Literal["_swhid",b"_swhid"]) -> typing.Optional[typing_extensions.Literal["swhid"]]: ... + swhid: builtins.str | None = ..., + label: collections.abc.Iterable[global___EdgeLabel] | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_swhid", b"_swhid", "swhid", b"swhid"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_swhid", b"_swhid", "label", b"label", "swhid", b"swhid"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_swhid", b"_swhid"]) -> typing_extensions.Literal["swhid"] | None: ... + global___Successor = Successor class ContentData(google.protobuf.message.Message): """Content node properties""" + DESCRIPTOR: google.protobuf.descriptor.Descriptor + LENGTH_FIELD_NUMBER: builtins.int IS_SKIPPED_FIELD_NUMBER: builtins.int length: builtins.int """Length of the blob, in bytes""" - is_skipped: builtins.bool """Whether the content was skipped during ingestion.""" - - def __init__(self, + def __init__( + self, *, - length: typing.Optional[builtins.int] = ..., - is_skipped: typing.Optional[builtins.bool] = ..., - ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_is_skipped",b"_is_skipped","_length",b"_length","is_skipped",b"is_skipped","length",b"length"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_is_skipped",b"_is_skipped","_length",b"_length","is_skipped",b"is_skipped","length",b"length"]) -> None: ... + length: builtins.int | None = ..., + is_skipped: builtins.bool | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_is_skipped", b"_is_skipped", "_length", b"_length", "is_skipped", b"is_skipped", "length", b"length"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_is_skipped", b"_is_skipped", "_length", b"_length", "is_skipped", b"is_skipped", "length", b"length"]) -> None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_is_skipped",b"_is_skipped"]) -> typing.Optional[typing_extensions.Literal["is_skipped"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_is_skipped", b"_is_skipped"]) -> typing_extensions.Literal["is_skipped"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_length",b"_length"]) -> typing.Optional[typing_extensions.Literal["length"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_length", b"_length"]) -> typing_extensions.Literal["length"] | None: ... + global___ContentData = ContentData class RevisionData(google.protobuf.message.Message): """Revision node properties""" + DESCRIPTOR: google.protobuf.descriptor.Descriptor + AUTHOR_FIELD_NUMBER: builtins.int AUTHOR_DATE_FIELD_NUMBER: builtins.int AUTHOR_DATE_OFFSET_FIELD_NUMBER: builtins.int COMMITTER_FIELD_NUMBER: builtins.int COMMITTER_DATE_FIELD_NUMBER: builtins.int COMMITTER_DATE_OFFSET_FIELD_NUMBER: builtins.int MESSAGE_FIELD_NUMBER: builtins.int author: builtins.int """Revision author ID (anonymized)""" - author_date: builtins.int """UNIX timestamp of the revision date (UTC)""" - author_date_offset: builtins.int """Timezone of the revision author date as an offset from UTC""" - committer: builtins.int """Revision committer ID (anonymized)""" - committer_date: builtins.int """UNIX timestamp of the revision committer date (UTC)""" - committer_date_offset: builtins.int """Timezone of the revision committer date as an offset from UTC""" - message: builtins.bytes """Revision message""" - - def __init__(self, + def __init__( + self, *, - author: typing.Optional[builtins.int] = ..., - author_date: typing.Optional[builtins.int] = ..., - author_date_offset: typing.Optional[builtins.int] = ..., - committer: typing.Optional[builtins.int] = ..., - committer_date: typing.Optional[builtins.int] = ..., - committer_date_offset: typing.Optional[builtins.int] = ..., - message: typing.Optional[builtins.bytes] = ..., - ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_author",b"_author","_author_date",b"_author_date","_author_date_offset",b"_author_date_offset","_committer",b"_committer","_committer_date",b"_committer_date","_committer_date_offset",b"_committer_date_offset","_message",b"_message","author",b"author","author_date",b"author_date","author_date_offset",b"author_date_offset","committer",b"committer","committer_date",b"committer_date","committer_date_offset",b"committer_date_offset","message",b"message"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_author",b"_author","_author_date",b"_author_date","_author_date_offset",b"_author_date_offset","_committer",b"_committer","_committer_date",b"_committer_date","_committer_date_offset",b"_committer_date_offset","_message",b"_message","author",b"author","author_date",b"author_date","author_date_offset",b"author_date_offset","committer",b"committer","committer_date",b"committer_date","committer_date_offset",b"committer_date_offset","message",b"message"]) -> None: ... + author: builtins.int | None = ..., + author_date: builtins.int | None = ..., + author_date_offset: builtins.int | None = ..., + committer: builtins.int | None = ..., + committer_date: builtins.int | None = ..., + committer_date_offset: builtins.int | None = ..., + message: builtins.bytes | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_author", b"_author", "_author_date", b"_author_date", "_author_date_offset", b"_author_date_offset", "_committer", b"_committer", "_committer_date", b"_committer_date", "_committer_date_offset", b"_committer_date_offset", "_message", b"_message", "author", b"author", "author_date", b"author_date", "author_date_offset", b"author_date_offset", "committer", b"committer", "committer_date", b"committer_date", "committer_date_offset", b"committer_date_offset", "message", b"message"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_author", b"_author", "_author_date", b"_author_date", "_author_date_offset", b"_author_date_offset", "_committer", b"_committer", "_committer_date", b"_committer_date", "_committer_date_offset", b"_committer_date_offset", "_message", b"_message", "author", b"author", "author_date", b"author_date", "author_date_offset", b"author_date_offset", "committer", b"committer", "committer_date", b"committer_date", "committer_date_offset", b"committer_date_offset", "message", b"message"]) -> None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_author",b"_author"]) -> typing.Optional[typing_extensions.Literal["author"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_author", b"_author"]) -> typing_extensions.Literal["author"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_author_date",b"_author_date"]) -> typing.Optional[typing_extensions.Literal["author_date"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_author_date", b"_author_date"]) -> typing_extensions.Literal["author_date"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_author_date_offset",b"_author_date_offset"]) -> typing.Optional[typing_extensions.Literal["author_date_offset"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_author_date_offset", b"_author_date_offset"]) -> typing_extensions.Literal["author_date_offset"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_committer",b"_committer"]) -> typing.Optional[typing_extensions.Literal["committer"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_committer", b"_committer"]) -> typing_extensions.Literal["committer"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_committer_date",b"_committer_date"]) -> typing.Optional[typing_extensions.Literal["committer_date"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_committer_date", b"_committer_date"]) -> typing_extensions.Literal["committer_date"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_committer_date_offset",b"_committer_date_offset"]) -> typing.Optional[typing_extensions.Literal["committer_date_offset"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_committer_date_offset", b"_committer_date_offset"]) -> typing_extensions.Literal["committer_date_offset"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_message",b"_message"]) -> typing.Optional[typing_extensions.Literal["message"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_message", b"_message"]) -> typing_extensions.Literal["message"] | None: ... + global___RevisionData = RevisionData class ReleaseData(google.protobuf.message.Message): """Release node properties""" + DESCRIPTOR: google.protobuf.descriptor.Descriptor + AUTHOR_FIELD_NUMBER: builtins.int AUTHOR_DATE_FIELD_NUMBER: builtins.int AUTHOR_DATE_OFFSET_FIELD_NUMBER: builtins.int NAME_FIELD_NUMBER: builtins.int MESSAGE_FIELD_NUMBER: builtins.int author: builtins.int """Release author ID (anonymized)""" - author_date: builtins.int """UNIX timestamp of the release date (UTC)""" - author_date_offset: builtins.int """Timezone of the release author date as an offset from UTC""" - name: builtins.bytes """Release name""" - message: builtins.bytes """Release message""" - - def __init__(self, + def __init__( + self, *, - author: typing.Optional[builtins.int] = ..., - author_date: typing.Optional[builtins.int] = ..., - author_date_offset: typing.Optional[builtins.int] = ..., - name: typing.Optional[builtins.bytes] = ..., - message: typing.Optional[builtins.bytes] = ..., - ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_author",b"_author","_author_date",b"_author_date","_author_date_offset",b"_author_date_offset","_message",b"_message","_name",b"_name","author",b"author","author_date",b"author_date","author_date_offset",b"author_date_offset","message",b"message","name",b"name"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_author",b"_author","_author_date",b"_author_date","_author_date_offset",b"_author_date_offset","_message",b"_message","_name",b"_name","author",b"author","author_date",b"author_date","author_date_offset",b"author_date_offset","message",b"message","name",b"name"]) -> None: ... + author: builtins.int | None = ..., + author_date: builtins.int | None = ..., + author_date_offset: builtins.int | None = ..., + name: builtins.bytes | None = ..., + message: builtins.bytes | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_author", b"_author", "_author_date", b"_author_date", "_author_date_offset", b"_author_date_offset", "_message", b"_message", "_name", b"_name", "author", b"author", "author_date", b"author_date", "author_date_offset", b"author_date_offset", "message", b"message", "name", b"name"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_author", b"_author", "_author_date", b"_author_date", "_author_date_offset", b"_author_date_offset", "_message", b"_message", "_name", b"_name", "author", b"author", "author_date", b"author_date", "author_date_offset", b"author_date_offset", "message", b"message", "name", b"name"]) -> None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_author",b"_author"]) -> typing.Optional[typing_extensions.Literal["author"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_author", b"_author"]) -> typing_extensions.Literal["author"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_author_date",b"_author_date"]) -> typing.Optional[typing_extensions.Literal["author_date"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_author_date", b"_author_date"]) -> typing_extensions.Literal["author_date"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_author_date_offset",b"_author_date_offset"]) -> typing.Optional[typing_extensions.Literal["author_date_offset"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_author_date_offset", b"_author_date_offset"]) -> typing_extensions.Literal["author_date_offset"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_message",b"_message"]) -> typing.Optional[typing_extensions.Literal["message"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_message", b"_message"]) -> typing_extensions.Literal["message"] | None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_name",b"_name"]) -> typing.Optional[typing_extensions.Literal["name"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_name", b"_name"]) -> typing_extensions.Literal["name"] | None: ... + global___ReleaseData = ReleaseData class OriginData(google.protobuf.message.Message): """Origin node properties""" + DESCRIPTOR: google.protobuf.descriptor.Descriptor + URL_FIELD_NUMBER: builtins.int - url: typing.Text + url: builtins.str """URL of the origin""" - - def __init__(self, + def __init__( + self, *, - url: typing.Optional[typing.Text] = ..., - ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_url",b"_url","url",b"url"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_url",b"_url","url",b"url"]) -> None: ... - def WhichOneof(self, oneof_group: typing_extensions.Literal["_url",b"_url"]) -> typing.Optional[typing_extensions.Literal["url"]]: ... + url: builtins.str | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_url", b"_url", "url", b"url"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_url", b"_url", "url", b"url"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_url", b"_url"]) -> typing_extensions.Literal["url"] | None: ... + global___OriginData = OriginData class EdgeLabel(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor + NAME_FIELD_NUMBER: builtins.int PERMISSION_FIELD_NUMBER: builtins.int name: builtins.bytes """Directory entry name for directories, branch name for snapshots""" - permission: builtins.int """Entry permission (only set for directories).""" - - def __init__(self, + def __init__( + self, *, name: builtins.bytes = ..., permission: builtins.int = ..., - ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["name",b"name","permission",b"permission"]) -> None: ... + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["name", b"name", "permission", b"permission"]) -> None: ... + global___EdgeLabel = EdgeLabel class CountResponse(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor + COUNT_FIELD_NUMBER: builtins.int count: builtins.int - def __init__(self, + def __init__( + self, *, count: builtins.int = ..., - ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["count",b"count"]) -> None: ... + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["count", b"count"]) -> None: ... + global___CountResponse = CountResponse class StatsRequest(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor - def __init__(self, - ) -> None: ... + + def __init__( + self, + ) -> None: ... + global___StatsRequest = StatsRequest class StatsResponse(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor + NUM_NODES_FIELD_NUMBER: builtins.int NUM_EDGES_FIELD_NUMBER: builtins.int COMPRESSION_RATIO_FIELD_NUMBER: builtins.int BITS_PER_NODE_FIELD_NUMBER: builtins.int BITS_PER_EDGE_FIELD_NUMBER: builtins.int AVG_LOCALITY_FIELD_NUMBER: builtins.int INDEGREE_MIN_FIELD_NUMBER: builtins.int INDEGREE_MAX_FIELD_NUMBER: builtins.int INDEGREE_AVG_FIELD_NUMBER: builtins.int OUTDEGREE_MIN_FIELD_NUMBER: builtins.int OUTDEGREE_MAX_FIELD_NUMBER: builtins.int OUTDEGREE_AVG_FIELD_NUMBER: builtins.int num_nodes: builtins.int """Number of nodes in the graph""" - num_edges: builtins.int """Number of edges in the graph""" - compression_ratio: builtins.float """Ratio between the graph size and the information-theoretical lower bound """ - bits_per_node: builtins.float """Number of bits per node (overall graph size in bits divided by the number of nodes) """ - bits_per_edge: builtins.float """Number of bits per edge (overall graph size in bits divided by the number of arcs). """ - avg_locality: builtins.float indegree_min: builtins.int """Smallest indegree""" - indegree_max: builtins.int """Largest indegree""" - indegree_avg: builtins.float """Average indegree""" - outdegree_min: builtins.int """Smallest outdegree""" - outdegree_max: builtins.int """Largest outdegree""" - outdegree_avg: builtins.float """Average outdegree""" - - def __init__(self, + def __init__( + self, *, num_nodes: builtins.int = ..., num_edges: builtins.int = ..., compression_ratio: builtins.float = ..., bits_per_node: builtins.float = ..., bits_per_edge: builtins.float = ..., avg_locality: builtins.float = ..., indegree_min: builtins.int = ..., indegree_max: builtins.int = ..., indegree_avg: builtins.float = ..., outdegree_min: builtins.int = ..., outdegree_max: builtins.int = ..., outdegree_avg: builtins.float = ..., - ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["avg_locality",b"avg_locality","bits_per_edge",b"bits_per_edge","bits_per_node",b"bits_per_node","compression_ratio",b"compression_ratio","indegree_avg",b"indegree_avg","indegree_max",b"indegree_max","indegree_min",b"indegree_min","num_edges",b"num_edges","num_nodes",b"num_nodes","outdegree_avg",b"outdegree_avg","outdegree_max",b"outdegree_max","outdegree_min",b"outdegree_min"]) -> None: ... + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["avg_locality", b"avg_locality", "bits_per_edge", b"bits_per_edge", "bits_per_node", b"bits_per_node", "compression_ratio", b"compression_ratio", "indegree_avg", b"indegree_avg", "indegree_max", b"indegree_max", "indegree_min", b"indegree_min", "num_edges", b"num_edges", "num_nodes", b"num_nodes", "outdegree_avg", b"outdegree_avg", "outdegree_max", b"outdegree_max", "outdegree_min", b"outdegree_min"]) -> None: ... + global___StatsResponse = StatsResponse diff --git a/swh/graph/rpc/swhgraph_pb2_grpc.py b/swh/graph/grpc/swhgraph_pb2_grpc.py similarity index 70% rename from swh/graph/rpc/swhgraph_pb2_grpc.py rename to swh/graph/grpc/swhgraph_pb2_grpc.py index ff60903..514e224 100644 --- a/swh/graph/rpc/swhgraph_pb2_grpc.py +++ b/swh/graph/grpc/swhgraph_pb2_grpc.py @@ -1,303 +1,303 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc -from swh.graph.rpc import swhgraph_pb2 as swh_dot_graph_dot_rpc_dot_swhgraph__pb2 +from swh.graph.grpc import swhgraph_pb2 as swh_dot_graph_dot_grpc_dot_swhgraph__pb2 class TraversalServiceStub(object): """Graph traversal service """ def __init__(self, channel): """Constructor. Args: channel: A grpc.Channel. """ self.GetNode = channel.unary_unary( '/swh.graph.TraversalService/GetNode', - request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.GetNodeRequest.SerializeToString, - response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.FromString, + request_serializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.GetNodeRequest.SerializeToString, + response_deserializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.Node.FromString, ) self.Traverse = channel.unary_stream( '/swh.graph.TraversalService/Traverse', - request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, - response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.FromString, + request_serializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, + response_deserializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.Node.FromString, ) self.FindPathTo = channel.unary_unary( '/swh.graph.TraversalService/FindPathTo', - request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.FindPathToRequest.SerializeToString, - response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Path.FromString, + request_serializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.FindPathToRequest.SerializeToString, + response_deserializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.Path.FromString, ) self.FindPathBetween = channel.unary_unary( '/swh.graph.TraversalService/FindPathBetween', - request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.FindPathBetweenRequest.SerializeToString, - response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Path.FromString, + request_serializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.FindPathBetweenRequest.SerializeToString, + response_deserializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.Path.FromString, ) self.CountNodes = channel.unary_unary( '/swh.graph.TraversalService/CountNodes', - request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, - response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.FromString, + request_serializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, + response_deserializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.CountResponse.FromString, ) self.CountEdges = channel.unary_unary( '/swh.graph.TraversalService/CountEdges', - request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, - response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.FromString, + request_serializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, + response_deserializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.CountResponse.FromString, ) self.Stats = channel.unary_unary( '/swh.graph.TraversalService/Stats', - request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsRequest.SerializeToString, - response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsResponse.FromString, + request_serializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.StatsRequest.SerializeToString, + response_deserializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.StatsResponse.FromString, ) class TraversalServiceServicer(object): """Graph traversal service """ def GetNode(self, request, context): """GetNode returns a single Node and its properties. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def Traverse(self, request, context): """Traverse performs a breadth-first graph traversal from a set of source nodes, then streams the nodes it encounters (if they match a given return filter), along with their properties. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def FindPathTo(self, request, context): """FindPathTo searches for a shortest path between a set of source nodes and a node that matches a specific *criteria*. It does so by performing a breadth-first search from the source node, until any node that matches the given criteria is found, then follows back its parents to return a shortest path from the source set to that node. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def FindPathBetween(self, request, context): """FindPathBetween searches for a shortest path between a set of source nodes and a set of destination nodes. It does so by performing a *bidirectional breadth-first search*, i.e., two parallel breadth-first searches, one from the source set ("src-BFS") and one from the destination set ("dst-BFS"), until both searches find a common node that joins their visited sets. This node is called the "midpoint node". The path returned is the path src -> ... -> midpoint -> ... -> dst, which is always a shortest path between src and dst. The graph direction of both BFS can be configured separately. By default, the dst-BFS will use the graph in the opposite direction than the src-BFS (if direction = FORWARD, by default direction_reverse = BACKWARD, and vice-versa). The default behavior is thus to search for a shortest path between two nodes in a given direction. However, one can also specify FORWARD or BACKWARD for *both* the src-BFS and the dst-BFS. This will search for a common descendant or a common ancestor between the two sets, respectively. These will be the midpoints of the returned path. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def CountNodes(self, request, context): """CountNodes does the same as Traverse, but only returns the number of nodes accessed during the traversal. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def CountEdges(self, request, context): """CountEdges does the same as Traverse, but only returns the number of edges accessed during the traversal. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def Stats(self, request, context): """Stats returns various statistics on the overall graph. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def add_TraversalServiceServicer_to_server(servicer, server): rpc_method_handlers = { 'GetNode': grpc.unary_unary_rpc_method_handler( servicer.GetNode, - request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.GetNodeRequest.FromString, - response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.SerializeToString, + request_deserializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.GetNodeRequest.FromString, + response_serializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.Node.SerializeToString, ), 'Traverse': grpc.unary_stream_rpc_method_handler( servicer.Traverse, - request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.FromString, - response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.SerializeToString, + request_deserializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.TraversalRequest.FromString, + response_serializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.Node.SerializeToString, ), 'FindPathTo': grpc.unary_unary_rpc_method_handler( servicer.FindPathTo, - request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.FindPathToRequest.FromString, - response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Path.SerializeToString, + request_deserializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.FindPathToRequest.FromString, + response_serializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.Path.SerializeToString, ), 'FindPathBetween': grpc.unary_unary_rpc_method_handler( servicer.FindPathBetween, - request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.FindPathBetweenRequest.FromString, - response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Path.SerializeToString, + request_deserializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.FindPathBetweenRequest.FromString, + response_serializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.Path.SerializeToString, ), 'CountNodes': grpc.unary_unary_rpc_method_handler( servicer.CountNodes, - request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.FromString, - response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.SerializeToString, + request_deserializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.TraversalRequest.FromString, + response_serializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.CountResponse.SerializeToString, ), 'CountEdges': grpc.unary_unary_rpc_method_handler( servicer.CountEdges, - request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.FromString, - response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.SerializeToString, + request_deserializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.TraversalRequest.FromString, + response_serializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.CountResponse.SerializeToString, ), 'Stats': grpc.unary_unary_rpc_method_handler( servicer.Stats, - request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsRequest.FromString, - response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsResponse.SerializeToString, + request_deserializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.StatsRequest.FromString, + response_serializer=swh_dot_graph_dot_grpc_dot_swhgraph__pb2.StatsResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( 'swh.graph.TraversalService', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) # This class is part of an EXPERIMENTAL API. class TraversalService(object): """Graph traversal service """ @staticmethod def GetNode(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/GetNode', - swh_dot_graph_dot_rpc_dot_swhgraph__pb2.GetNodeRequest.SerializeToString, - swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.FromString, + swh_dot_graph_dot_grpc_dot_swhgraph__pb2.GetNodeRequest.SerializeToString, + swh_dot_graph_dot_grpc_dot_swhgraph__pb2.Node.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod def Traverse(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_stream(request, target, '/swh.graph.TraversalService/Traverse', - swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, - swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.FromString, + swh_dot_graph_dot_grpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, + swh_dot_graph_dot_grpc_dot_swhgraph__pb2.Node.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod def FindPathTo(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/FindPathTo', - swh_dot_graph_dot_rpc_dot_swhgraph__pb2.FindPathToRequest.SerializeToString, - swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Path.FromString, + swh_dot_graph_dot_grpc_dot_swhgraph__pb2.FindPathToRequest.SerializeToString, + swh_dot_graph_dot_grpc_dot_swhgraph__pb2.Path.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod def FindPathBetween(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/FindPathBetween', - swh_dot_graph_dot_rpc_dot_swhgraph__pb2.FindPathBetweenRequest.SerializeToString, - swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Path.FromString, + swh_dot_graph_dot_grpc_dot_swhgraph__pb2.FindPathBetweenRequest.SerializeToString, + swh_dot_graph_dot_grpc_dot_swhgraph__pb2.Path.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod def CountNodes(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/CountNodes', - swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, - swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.FromString, + swh_dot_graph_dot_grpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, + swh_dot_graph_dot_grpc_dot_swhgraph__pb2.CountResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod def CountEdges(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/CountEdges', - swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, - swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.FromString, + swh_dot_graph_dot_grpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, + swh_dot_graph_dot_grpc_dot_swhgraph__pb2.CountResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod def Stats(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/Stats', - swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsRequest.SerializeToString, - swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsResponse.FromString, + swh_dot_graph_dot_grpc_dot_swhgraph__pb2.StatsRequest.SerializeToString, + swh_dot_graph_dot_grpc_dot_swhgraph__pb2.StatsResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/swh/graph/rpc_server.py b/swh/graph/grpc_server.py similarity index 67% rename from swh/graph/rpc_server.py rename to swh/graph/grpc_server.py index f6e1b4b..5c129a5 100644 --- a/swh/graph/rpc_server.py +++ b/swh/graph/grpc_server.py @@ -1,48 +1,54 @@ -# Copyright (C) 2021 The Software Heritage developers +# Copyright (C) 2021-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """ A simple tool to start the swh-graph GRPC server in Java. """ import logging import shlex import subprocess import aiohttp.test_utils import aiohttp.web from swh.graph.config import check_config -def spawn_java_rpc_server(config, port=None): +def build_grpc_server_cmdline(**config): + port = config.pop("port", None) if port is None: port = aiohttp.test_utils.unused_port() - config = check_config(config or {}) + config = check_config(config) cmd = [ "java", - "-cp", + "--class-path", config["classpath"], *config["java_tool_options"].split(), "org.softwareheritage.graph.rpc.GraphServer", "--port", str(port), - str(config["graph"]["path"]), + str(config["path"]), ] + return cmd, port + + +def spawn_java_grpc_server(**config): + cmd, port = build_grpc_server_cmdline(**config) print(cmd) # XXX: shlex.join() is in 3.8 # logging.info("Starting RPC server: %s", shlex.join(cmd)) - logging.info("Starting RPC server: %s", " ".join(shlex.quote(x) for x in cmd)) + logging.info("Starting GRPC server: %s", " ".join(shlex.quote(x) for x in cmd)) server = subprocess.Popen(cmd) return server, port -def stop_java_rpc_server(server: subprocess.Popen, timeout: int = 15): +def stop_java_grpc_server(server: subprocess.Popen, timeout: int = 15): server.terminate() try: server.wait(timeout=timeout) except subprocess.TimeoutExpired: logging.warning("Server did not terminate, sending kill signal...") server.kill() diff --git a/swh/graph/http_server.py b/swh/graph/http_rpc_server.py similarity index 87% rename from swh/graph/http_server.py rename to swh/graph/http_rpc_server.py index 53cf1bc..658a4ea 100644 --- a/swh/graph/http_server.py +++ b/swh/graph/http_rpc_server.py @@ -1,372 +1,415 @@ # Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """ A proxy HTTP server for swh-graph, talking to the Java code via py4j, and using FIFO as a transport to stream integers between the two languages. """ import json +import logging import os from typing import Optional import aiohttp.test_utils import aiohttp.web from google.protobuf import json_format from google.protobuf.field_mask_pb2 import FieldMask import grpc from swh.core.api.asynchronous import RPCServerApp from swh.core.config import read as config_read -from swh.graph.rpc.swhgraph_pb2 import ( +from swh.graph.grpc.swhgraph_pb2 import ( GetNodeRequest, NodeFilter, StatsRequest, TraversalRequest, ) -from swh.graph.rpc.swhgraph_pb2_grpc import TraversalServiceStub -from swh.graph.rpc_server import spawn_java_rpc_server, stop_java_rpc_server +from swh.graph.grpc.swhgraph_pb2_grpc import TraversalServiceStub +from swh.graph.grpc_server import spawn_java_grpc_server, stop_java_grpc_server from swh.model.swhids import EXTENDED_SWHID_TYPES try: from contextlib import asynccontextmanager except ImportError: # Compatibility with 3.6 backport from async_generator import asynccontextmanager # type: ignore # maximum number of retries for random walks RANDOM_RETRIES = 10 # TODO make this configurable via rpc-serve configuration +logger = logging.getLogger(__name__) + async def _aiorpcerror_middleware(app, handler): async def middleware_handler(request): try: return await handler(request) except grpc.aio.AioRpcError as e: # The default error handler of the RPC framework tries to serialize this # with msgpack; which for some unknown reason causes it to raise # ValueError("recursion limit exceeded") with a lot of context, causing # Sentry to be overflowed with gigabytes of logs (160KB per event, with # potentially hundreds of thousands of events per day). # Instead, we simply serialize the exception to a string. # https://sentry.softwareheritage.org/share/issue/d6d4db971e4b47728a6c1dd06cb9b8a5/ raise aiohttp.web.HTTPServiceUnavailable(text=str(e)) return middleware_handler class GraphServerApp(RPCServerApp): def __init__(self, *args, middlewares=(), **kwargs): middlewares = (_aiorpcerror_middleware,) + middlewares super().__init__(*args, middlewares=middlewares, **kwargs) self.on_startup.append(self._start) self.on_shutdown.append(self._stop) @staticmethod async def _start(app): app["channel"] = grpc.aio.insecure_channel(app["rpc_url"]) await app["channel"].__aenter__() app["rpc_client"] = TraversalServiceStub(app["channel"]) await app["rpc_client"].Stats(StatsRequest(), wait_for_ready=True) @staticmethod async def _stop(app): await app["channel"].__aexit__(None, None, None) if app.get("local_server"): - stop_java_rpc_server(app["local_server"]) + stop_java_grpc_server(app["local_server"]) async def index(request): return aiohttp.web.Response( content_type="text/html", body=""" Software Heritage graph server

You have reached the Software Heritage graph API server.

See its API documentation for more information.

""", ) class GraphView(aiohttp.web.View): """Base class for views working on the graph, with utility functions""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.rpc_client: TraversalServiceStub = self.request.app["rpc_client"] def get_direction(self): """Validate HTTP query parameter `direction`""" s = self.request.query.get("direction", "forward") if s not in ("forward", "backward"): raise aiohttp.web.HTTPBadRequest(text=f"invalid direction: {s}") return s.upper() def get_edges(self): """Validate HTTP query parameter `edges`, i.e., edge restrictions""" s = self.request.query.get("edges", "*") if any( [ node_type != "*" and node_type not in EXTENDED_SWHID_TYPES for edge in s.split(":") for node_type in edge.split(",", maxsplit=1) ] ): raise aiohttp.web.HTTPBadRequest(text=f"invalid edge restriction: {s}") return s def get_return_types(self): """Validate HTTP query parameter 'return types', i.e, a set of types which we will filter the query results with""" s = self.request.query.get("return_types", "*") if any( node_type != "*" and node_type not in EXTENDED_SWHID_TYPES for node_type in s.split(",") ): raise aiohttp.web.HTTPBadRequest( text=f"invalid type for filtering res: {s}" ) # if the user puts a star, # then we filter nothing, we don't need the other information if "*" in s: return "*" else: return s - def get_traversal(self): - """Validate HTTP query parameter `traversal`, i.e., visit order""" - s = self.request.query.get("traversal", "dfs") - if s not in ("bfs", "dfs"): - raise aiohttp.web.HTTPBadRequest(text=f"invalid traversal order: {s}") - return s - def get_limit(self): """Validate HTTP query parameter `limit`, i.e., number of results""" s = self.request.query.get("limit", "0") try: return int(s) except ValueError: raise aiohttp.web.HTTPBadRequest(text=f"invalid limit value: {s}") def get_max_edges(self): """Validate HTTP query parameter 'max_edges', i.e., the limit of the number of edges that can be visited""" s = self.request.query.get("max_edges", "0") try: return int(s) except ValueError: raise aiohttp.web.HTTPBadRequest(text=f"invalid max_edges value: {s}") async def check_swhid(self, swhid): """Validate that the given SWHID exists in the graph""" try: await self.rpc_client.GetNode( GetNodeRequest(swhid=swhid, mask=FieldMask(paths=["swhid"])) ) except grpc.aio.AioRpcError as e: if e.code() == grpc.StatusCode.INVALID_ARGUMENT: raise aiohttp.web.HTTPBadRequest(text=str(e.details())) class StreamingGraphView(GraphView): """Base class for views streaming their response line by line.""" content_type = "text/plain" @asynccontextmanager async def response_streamer(self, *args, **kwargs): """Context manager to prepare then close a StreamResponse""" response = aiohttp.web.StreamResponse(*args, **kwargs) response.content_type = self.content_type await response.prepare(self.request) yield response await response.write_eof() async def get(self): await self.prepare_response() async with self.response_streamer() as self.response_stream: self._buf = [] try: await self.stream_response() finally: await self._flush_buffer() return self.response_stream async def prepare_response(self): """This can be overridden with some setup to be run before the response actually starts streaming. """ pass async def stream_response(self): """Override this to perform the response streaming. Implementations of this should await self.stream_line(line) to write each line. """ raise NotImplementedError async def stream_line(self, line): """Write a line in the response stream.""" self._buf.append(line) if len(self._buf) > 100: await self._flush_buffer() async def _flush_buffer(self): await self.response_stream.write("\n".join(self._buf).encode() + b"\n") self._buf = [] class StatsView(GraphView): """View showing some statistics on the graph""" async def get(self): res = await self.rpc_client.Stats(StatsRequest()) stats = json_format.MessageToDict( res, including_default_value_fields=True, preserving_proto_field_name=True ) # Int64 fields are serialized as strings by default. for descriptor in res.DESCRIPTOR.fields: if descriptor.type == descriptor.TYPE_INT64: try: stats[descriptor.name] = int(stats[descriptor.name]) except KeyError: pass json_body = json.dumps(stats, indent=4, sort_keys=True) return aiohttp.web.Response(body=json_body, content_type="application/json") class SimpleTraversalView(StreamingGraphView): """Base class for views of simple traversals""" async def prepare_response(self): src = self.request.match_info["src"] self.traversal_request = TraversalRequest( src=[src], edges=self.get_edges(), direction=self.get_direction(), return_nodes=NodeFilter(types=self.get_return_types()), mask=FieldMask(paths=["swhid"]), ) if self.get_max_edges(): self.traversal_request.max_edges = self.get_max_edges() await self.check_swhid(src) self.configure_request() self.nodes_stream = self.rpc_client.Traverse(self.traversal_request) # Force gRPC to query the server and fetch the first nodes; so errors # are raised early, so we can return HTTP 503 before HTTP 200 await self.nodes_stream.wait_for_connection() def configure_request(self): pass async def stream_response(self): async for node in self.nodes_stream: await self.stream_line(node.swhid) class LeavesView(SimpleTraversalView): def configure_request(self): self.traversal_request.return_nodes.max_traversal_successors = 0 class NeighborsView(SimpleTraversalView): def configure_request(self): self.traversal_request.min_depth = 1 self.traversal_request.max_depth = 1 class VisitNodesView(SimpleTraversalView): pass class VisitEdgesView(SimpleTraversalView): def configure_request(self): self.traversal_request.mask.paths.extend(["successor", "successor.swhid"]) # self.traversal_request.return_fields.successor = True async def stream_response(self): async for node in self.rpc_client.Traverse(self.traversal_request): for succ in node.successor: await self.stream_line(node.swhid + " " + succ.swhid) class CountView(GraphView): """Base class for counting views.""" count_type: Optional[str] = None async def get(self): src = self.request.match_info["src"] self.traversal_request = TraversalRequest( src=[src], edges=self.get_edges(), direction=self.get_direction(), return_nodes=NodeFilter(types=self.get_return_types()), mask=FieldMask(paths=["swhid"]), ) if self.get_max_edges(): self.traversal_request.max_edges = self.get_max_edges() self.configure_request() res = await self.rpc_client.CountNodes(self.traversal_request) return aiohttp.web.Response( body=str(res.count), content_type="application/json" ) def configure_request(self): pass class CountNeighborsView(CountView): def configure_request(self): self.traversal_request.min_depth = 1 self.traversal_request.max_depth = 1 class CountLeavesView(CountView): def configure_request(self): self.traversal_request.return_nodes.max_traversal_successors = 0 class CountVisitNodesView(CountView): pass -def make_app(config=None, rpc_url=None, spawn_rpc_port=50091, **kwargs): - app = GraphServerApp(**kwargs) +def make_app(config=None): + """Create an aiohttp server for the HTTP RPC frontend to the swh-graph API. - if rpc_url is None: - app["local_server"], port = spawn_java_rpc_server(config, port=spawn_rpc_port) - rpc_url = f"localhost:{port}" + It may either connect to an existing grpc server (cls="remote") or spawn a + local grpc server (cls="local"). + + ``config`` is expected to be a dict like:: + + graph: + cls: "local" + grpc_server: + port: 50091 + http_rpc_server: + debug: true + + or:: + graph: + cls: "remote" + url: "localhost:50091" + http_rpc_server: + debug: true + + See: + + - :mod:`swh.graph.grpc_server` for more details of the content of the + grpc_server section, + + - :class:`~.GraphServerApp` class for more details of the content of the + http_rpc_server section. + + """ + if config is None: + config = {} + if "graph" not in config: + logger.info( + "Missing 'graph' configuration; default to a locally spawn" + "grpc server listening on 0.0.0.0:50091" + ) + cfg = {"cls": "local", "grpc_server": {"port": 50091}} + else: + cfg = config["graph"].copy() + cls = cfg.pop("cls") + grpc_cfg = cfg.pop("grpc_server", {}) + app = GraphServerApp(**cfg.get("http_rpc_server", {})) + if cls == "remote": + if "url" not in cfg: + raise KeyError("Missing 'url' configuration entry in the [graph] section") + rpc_url = cfg["url"] + elif cls == "local": + app["local_server"], port = spawn_java_grpc_server(**grpc_cfg) + rpc_url = f"localhost:{port}" + else: + raise ValueError(f"Unknown swh.graph class cls={cls}") app.add_routes( [ aiohttp.web.get("/", index), aiohttp.web.get("/graph", index), aiohttp.web.view("/graph/stats", StatsView), aiohttp.web.view("/graph/leaves/{src}", LeavesView), aiohttp.web.view("/graph/neighbors/{src}", NeighborsView), aiohttp.web.view("/graph/visit/nodes/{src}", VisitNodesView), aiohttp.web.view("/graph/visit/edges/{src}", VisitEdgesView), aiohttp.web.view("/graph/neighbors/count/{src}", CountNeighborsView), aiohttp.web.view("/graph/leaves/count/{src}", CountLeavesView), aiohttp.web.view("/graph/visit/nodes/count/{src}", CountVisitNodesView), ] ) app["rpc_url"] = rpc_url return app def make_app_from_configfile(): """Load configuration and then build application to run""" config_file = os.environ.get("SWH_CONFIG_FILENAME") config = config_read(config_file) return make_app(config=config) diff --git a/swh/graph/tests/conftest.py b/swh/graph/pytest_plugin.py similarity index 85% rename from swh/graph/tests/conftest.py rename to swh/graph/pytest_plugin.py index 6e832af..77cb9d3 100644 --- a/swh/graph/tests/conftest.py +++ b/swh/graph/pytest_plugin.py @@ -1,107 +1,118 @@ # Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import logging import multiprocessing from pathlib import Path import subprocess from aiohttp.test_utils import TestClient, TestServer, loop_context import grpc import pytest +from swh.graph.grpc.swhgraph_pb2_grpc import TraversalServiceStub from swh.graph.http_client import RemoteGraphClient from swh.graph.http_naive_client import NaiveClient -from swh.graph.rpc.swhgraph_pb2_grpc import TraversalServiceStub -SWH_GRAPH_TESTS_ROOT = Path(__file__).parents[0] +SWH_GRAPH_TESTS_ROOT = Path(__file__).parents[0] / "tests" TEST_GRAPH_PATH = SWH_GRAPH_TESTS_ROOT / "dataset/compressed/example" +logger = logging.getLogger(__name__) + + class GraphServerProcess(multiprocessing.Process): def __init__(self, *args, **kwargs): self.q = multiprocessing.Queue() super().__init__(*args, **kwargs) def run(self): # Lazy import to allow debian packaging - from swh.graph.http_server import make_app + from swh.graph.http_rpc_server import make_app try: - config = {"graph": {"path": TEST_GRAPH_PATH}} + config = { + "graph": { + "cls": "local", + "grpc_server": {"path": TEST_GRAPH_PATH}, + "http_rpc_server": {"debug": True}, + } + } with loop_context() as loop: - app = make_app(config=config, debug=True, spawn_rpc_port=None) + app = make_app(config=config) client = TestClient(TestServer(app), loop=loop) loop.run_until_complete(client.start_server()) url = client.make_url("/graph/") self.q.put( { "server_url": url, "rpc_url": app["rpc_url"], "pid": app["local_server"].pid, } ) loop.run_forever() except Exception as e: + logger.exception(e) self.q.put(e) def start(self, *args, **kwargs): super().start() self.result = self.q.get() @pytest.fixture(scope="module") def graph_grpc_server_process(): server = GraphServerProcess() yield server server.kill() @pytest.fixture(scope="module") def graph_grpc_server(graph_grpc_server_process): server = graph_grpc_server_process server.start() if isinstance(server.result, Exception): raise server.result grpc_url = server.result["rpc_url"] yield grpc_url server.kill() @pytest.fixture(scope="module") def graph_grpc_stub(graph_grpc_server): with grpc.insecure_channel(graph_grpc_server) as channel: stub = TraversalServiceStub(channel) yield stub @pytest.fixture(scope="module", params=["remote", "naive"]) def graph_client(request): if request.param == "remote": server = request.getfixturevalue("graph_grpc_server_process") server.start() if isinstance(server.result, Exception): raise server.result yield RemoteGraphClient(str(server.result["server_url"])) server.kill() else: def zstdcat(*files): p = subprocess.run(["zstdcat", *files], stdout=subprocess.PIPE) return p.stdout.decode() edges_dataset = SWH_GRAPH_TESTS_ROOT / "dataset/edges" edge_files = edges_dataset.glob("*/*.edges.csv.zst") node_files = edges_dataset.glob("*/*.nodes.csv.zst") nodes = set(zstdcat(*node_files).strip().split("\n")) edge_lines = [line.split() for line in zstdcat(*edge_files).strip().split("\n")] edges = [(src, dst) for src, dst, *_ in edge_lines] for src, dst in edges: nodes.add(src) nodes.add(dst) yield NaiveClient(nodes=list(nodes), edges=edges) diff --git a/swh/graph/rpc/swhgraph_pb2.py b/swh/graph/rpc/swhgraph_pb2.py deleted file mode 100644 index b48ad97..0000000 --- a/swh/graph/rpc/swhgraph_pb2.py +++ /dev/null @@ -1,196 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: swh/graph/rpc/swhgraph.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import enum_type_wrapper -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from google.protobuf import field_mask_pb2 as google_dot_protobuf_dot_field__mask__pb2 - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cswh/graph/rpc/swhgraph.proto\x12\tswh.graph\x1a google/protobuf/field_mask.proto\"W\n\x0eGetNodeRequest\x12\r\n\x05swhid\x18\x01 \x01(\t\x12-\n\x04mask\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x00\x88\x01\x01\x42\x07\n\x05_mask\"\xd8\x02\n\x10TraversalRequest\x12\x0b\n\x03src\x18\x01 \x03(\t\x12,\n\tdirection\x18\x02 \x01(\x0e\x32\x19.swh.graph.GraphDirection\x12\x12\n\x05\x65\x64ges\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tmax_edges\x18\x04 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\tmin_depth\x18\x05 \x01(\x03H\x02\x88\x01\x01\x12\x16\n\tmax_depth\x18\x06 \x01(\x03H\x03\x88\x01\x01\x12\x30\n\x0creturn_nodes\x18\x07 \x01(\x0b\x32\x15.swh.graph.NodeFilterH\x04\x88\x01\x01\x12-\n\x04mask\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x05\x88\x01\x01\x42\x08\n\x06_edgesB\x0c\n\n_max_edgesB\x0c\n\n_min_depthB\x0c\n\n_max_depthB\x0f\n\r_return_nodesB\x07\n\x05_mask\"\x97\x02\n\x11\x46indPathToRequest\x12\x0b\n\x03src\x18\x01 \x03(\t\x12%\n\x06target\x18\x02 \x01(\x0b\x32\x15.swh.graph.NodeFilter\x12,\n\tdirection\x18\x03 \x01(\x0e\x32\x19.swh.graph.GraphDirection\x12\x12\n\x05\x65\x64ges\x18\x04 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tmax_edges\x18\x05 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\tmax_depth\x18\x06 \x01(\x03H\x02\x88\x01\x01\x12-\n\x04mask\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x03\x88\x01\x01\x42\x08\n\x06_edgesB\x0c\n\n_max_edgesB\x0c\n\n_max_depthB\x07\n\x05_mask\"\x81\x03\n\x16\x46indPathBetweenRequest\x12\x0b\n\x03src\x18\x01 \x03(\t\x12\x0b\n\x03\x64st\x18\x02 \x03(\t\x12,\n\tdirection\x18\x03 \x01(\x0e\x32\x19.swh.graph.GraphDirection\x12\x39\n\x11\x64irection_reverse\x18\x04 \x01(\x0e\x32\x19.swh.graph.GraphDirectionH\x00\x88\x01\x01\x12\x12\n\x05\x65\x64ges\x18\x05 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\redges_reverse\x18\x06 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tmax_edges\x18\x07 \x01(\x03H\x03\x88\x01\x01\x12\x16\n\tmax_depth\x18\x08 \x01(\x03H\x04\x88\x01\x01\x12-\n\x04mask\x18\t \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x05\x88\x01\x01\x42\x14\n\x12_direction_reverseB\x08\n\x06_edgesB\x10\n\x0e_edges_reverseB\x0c\n\n_max_edgesB\x0c\n\n_max_depthB\x07\n\x05_mask\"\xb2\x01\n\nNodeFilter\x12\x12\n\x05types\x18\x01 \x01(\tH\x00\x88\x01\x01\x12%\n\x18min_traversal_successors\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12%\n\x18max_traversal_successors\x18\x03 \x01(\x03H\x02\x88\x01\x01\x42\x08\n\x06_typesB\x1b\n\x19_min_traversal_successorsB\x1b\n\x19_max_traversal_successors\"\x92\x02\n\x04Node\x12\r\n\x05swhid\x18\x01 \x01(\t\x12\'\n\tsuccessor\x18\x02 \x03(\x0b\x32\x14.swh.graph.Successor\x12\x1b\n\x0enum_successors\x18\t \x01(\x03H\x01\x88\x01\x01\x12%\n\x03\x63nt\x18\x03 \x01(\x0b\x32\x16.swh.graph.ContentDataH\x00\x12&\n\x03rev\x18\x05 \x01(\x0b\x32\x17.swh.graph.RevisionDataH\x00\x12%\n\x03rel\x18\x06 \x01(\x0b\x32\x16.swh.graph.ReleaseDataH\x00\x12$\n\x03ori\x18\x08 \x01(\x0b\x32\x15.swh.graph.OriginDataH\x00\x42\x06\n\x04\x64\x61taB\x11\n\x0f_num_successors\"U\n\x04Path\x12\x1d\n\x04node\x18\x01 \x03(\x0b\x32\x0f.swh.graph.Node\x12\x1b\n\x0emidpoint_index\x18\x02 \x01(\x05H\x00\x88\x01\x01\x42\x11\n\x0f_midpoint_index\"N\n\tSuccessor\x12\x12\n\x05swhid\x18\x01 \x01(\tH\x00\x88\x01\x01\x12#\n\x05label\x18\x02 \x03(\x0b\x32\x14.swh.graph.EdgeLabelB\x08\n\x06_swhid\"U\n\x0b\x43ontentData\x12\x13\n\x06length\x18\x01 \x01(\x03H\x00\x88\x01\x01\x12\x17\n\nis_skipped\x18\x02 \x01(\x08H\x01\x88\x01\x01\x42\t\n\x07_lengthB\r\n\x0b_is_skipped\"\xc6\x02\n\x0cRevisionData\x12\x13\n\x06\x61uthor\x18\x01 \x01(\x03H\x00\x88\x01\x01\x12\x18\n\x0b\x61uthor_date\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1f\n\x12\x61uthor_date_offset\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x16\n\tcommitter\x18\x04 \x01(\x03H\x03\x88\x01\x01\x12\x1b\n\x0e\x63ommitter_date\x18\x05 \x01(\x03H\x04\x88\x01\x01\x12\"\n\x15\x63ommitter_date_offset\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x14\n\x07message\x18\x07 \x01(\x0cH\x06\x88\x01\x01\x42\t\n\x07_authorB\x0e\n\x0c_author_dateB\x15\n\x13_author_date_offsetB\x0c\n\n_committerB\x11\n\x0f_committer_dateB\x18\n\x16_committer_date_offsetB\n\n\x08_message\"\xcd\x01\n\x0bReleaseData\x12\x13\n\x06\x61uthor\x18\x01 \x01(\x03H\x00\x88\x01\x01\x12\x18\n\x0b\x61uthor_date\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1f\n\x12\x61uthor_date_offset\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x11\n\x04name\x18\x04 \x01(\x0cH\x03\x88\x01\x01\x12\x14\n\x07message\x18\x05 \x01(\x0cH\x04\x88\x01\x01\x42\t\n\x07_authorB\x0e\n\x0c_author_dateB\x15\n\x13_author_date_offsetB\x07\n\x05_nameB\n\n\x08_message\"&\n\nOriginData\x12\x10\n\x03url\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x06\n\x04_url\"-\n\tEdgeLabel\x12\x0c\n\x04name\x18\x01 \x01(\x0c\x12\x12\n\npermission\x18\x02 \x01(\x05\"\x1e\n\rCountResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\"\x0e\n\x0cStatsRequest\"\x9b\x02\n\rStatsResponse\x12\x11\n\tnum_nodes\x18\x01 \x01(\x03\x12\x11\n\tnum_edges\x18\x02 \x01(\x03\x12\x19\n\x11\x63ompression_ratio\x18\x03 \x01(\x01\x12\x15\n\rbits_per_node\x18\x04 \x01(\x01\x12\x15\n\rbits_per_edge\x18\x05 \x01(\x01\x12\x14\n\x0c\x61vg_locality\x18\x06 \x01(\x01\x12\x14\n\x0cindegree_min\x18\x07 \x01(\x03\x12\x14\n\x0cindegree_max\x18\x08 \x01(\x03\x12\x14\n\x0cindegree_avg\x18\t \x01(\x01\x12\x15\n\routdegree_min\x18\n \x01(\x03\x12\x15\n\routdegree_max\x18\x0b \x01(\x03\x12\x15\n\routdegree_avg\x18\x0c \x01(\x01*+\n\x0eGraphDirection\x12\x0b\n\x07\x46ORWARD\x10\x00\x12\x0c\n\x08\x42\x41\x43KWARD\x10\x01\x32\xcf\x03\n\x10TraversalService\x12\x35\n\x07GetNode\x12\x19.swh.graph.GetNodeRequest\x1a\x0f.swh.graph.Node\x12:\n\x08Traverse\x12\x1b.swh.graph.TraversalRequest\x1a\x0f.swh.graph.Node0\x01\x12;\n\nFindPathTo\x12\x1c.swh.graph.FindPathToRequest\x1a\x0f.swh.graph.Path\x12\x45\n\x0f\x46indPathBetween\x12!.swh.graph.FindPathBetweenRequest\x1a\x0f.swh.graph.Path\x12\x43\n\nCountNodes\x12\x1b.swh.graph.TraversalRequest\x1a\x18.swh.graph.CountResponse\x12\x43\n\nCountEdges\x12\x1b.swh.graph.TraversalRequest\x1a\x18.swh.graph.CountResponse\x12:\n\x05Stats\x12\x17.swh.graph.StatsRequest\x1a\x18.swh.graph.StatsResponseB0\n\x1eorg.softwareheritage.graph.rpcB\x0cGraphServiceP\x01\x62\x06proto3') - -_GRAPHDIRECTION = DESCRIPTOR.enum_types_by_name['GraphDirection'] -GraphDirection = enum_type_wrapper.EnumTypeWrapper(_GRAPHDIRECTION) -FORWARD = 0 -BACKWARD = 1 - - -_GETNODEREQUEST = DESCRIPTOR.message_types_by_name['GetNodeRequest'] -_TRAVERSALREQUEST = DESCRIPTOR.message_types_by_name['TraversalRequest'] -_FINDPATHTOREQUEST = DESCRIPTOR.message_types_by_name['FindPathToRequest'] -_FINDPATHBETWEENREQUEST = DESCRIPTOR.message_types_by_name['FindPathBetweenRequest'] -_NODEFILTER = DESCRIPTOR.message_types_by_name['NodeFilter'] -_NODE = DESCRIPTOR.message_types_by_name['Node'] -_PATH = DESCRIPTOR.message_types_by_name['Path'] -_SUCCESSOR = DESCRIPTOR.message_types_by_name['Successor'] -_CONTENTDATA = DESCRIPTOR.message_types_by_name['ContentData'] -_REVISIONDATA = DESCRIPTOR.message_types_by_name['RevisionData'] -_RELEASEDATA = DESCRIPTOR.message_types_by_name['ReleaseData'] -_ORIGINDATA = DESCRIPTOR.message_types_by_name['OriginData'] -_EDGELABEL = DESCRIPTOR.message_types_by_name['EdgeLabel'] -_COUNTRESPONSE = DESCRIPTOR.message_types_by_name['CountResponse'] -_STATSREQUEST = DESCRIPTOR.message_types_by_name['StatsRequest'] -_STATSRESPONSE = DESCRIPTOR.message_types_by_name['StatsResponse'] -GetNodeRequest = _reflection.GeneratedProtocolMessageType('GetNodeRequest', (_message.Message,), { - 'DESCRIPTOR' : _GETNODEREQUEST, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.GetNodeRequest) - }) -_sym_db.RegisterMessage(GetNodeRequest) - -TraversalRequest = _reflection.GeneratedProtocolMessageType('TraversalRequest', (_message.Message,), { - 'DESCRIPTOR' : _TRAVERSALREQUEST, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.TraversalRequest) - }) -_sym_db.RegisterMessage(TraversalRequest) - -FindPathToRequest = _reflection.GeneratedProtocolMessageType('FindPathToRequest', (_message.Message,), { - 'DESCRIPTOR' : _FINDPATHTOREQUEST, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.FindPathToRequest) - }) -_sym_db.RegisterMessage(FindPathToRequest) - -FindPathBetweenRequest = _reflection.GeneratedProtocolMessageType('FindPathBetweenRequest', (_message.Message,), { - 'DESCRIPTOR' : _FINDPATHBETWEENREQUEST, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.FindPathBetweenRequest) - }) -_sym_db.RegisterMessage(FindPathBetweenRequest) - -NodeFilter = _reflection.GeneratedProtocolMessageType('NodeFilter', (_message.Message,), { - 'DESCRIPTOR' : _NODEFILTER, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.NodeFilter) - }) -_sym_db.RegisterMessage(NodeFilter) - -Node = _reflection.GeneratedProtocolMessageType('Node', (_message.Message,), { - 'DESCRIPTOR' : _NODE, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.Node) - }) -_sym_db.RegisterMessage(Node) - -Path = _reflection.GeneratedProtocolMessageType('Path', (_message.Message,), { - 'DESCRIPTOR' : _PATH, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.Path) - }) -_sym_db.RegisterMessage(Path) - -Successor = _reflection.GeneratedProtocolMessageType('Successor', (_message.Message,), { - 'DESCRIPTOR' : _SUCCESSOR, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.Successor) - }) -_sym_db.RegisterMessage(Successor) - -ContentData = _reflection.GeneratedProtocolMessageType('ContentData', (_message.Message,), { - 'DESCRIPTOR' : _CONTENTDATA, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.ContentData) - }) -_sym_db.RegisterMessage(ContentData) - -RevisionData = _reflection.GeneratedProtocolMessageType('RevisionData', (_message.Message,), { - 'DESCRIPTOR' : _REVISIONDATA, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.RevisionData) - }) -_sym_db.RegisterMessage(RevisionData) - -ReleaseData = _reflection.GeneratedProtocolMessageType('ReleaseData', (_message.Message,), { - 'DESCRIPTOR' : _RELEASEDATA, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.ReleaseData) - }) -_sym_db.RegisterMessage(ReleaseData) - -OriginData = _reflection.GeneratedProtocolMessageType('OriginData', (_message.Message,), { - 'DESCRIPTOR' : _ORIGINDATA, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.OriginData) - }) -_sym_db.RegisterMessage(OriginData) - -EdgeLabel = _reflection.GeneratedProtocolMessageType('EdgeLabel', (_message.Message,), { - 'DESCRIPTOR' : _EDGELABEL, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.EdgeLabel) - }) -_sym_db.RegisterMessage(EdgeLabel) - -CountResponse = _reflection.GeneratedProtocolMessageType('CountResponse', (_message.Message,), { - 'DESCRIPTOR' : _COUNTRESPONSE, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.CountResponse) - }) -_sym_db.RegisterMessage(CountResponse) - -StatsRequest = _reflection.GeneratedProtocolMessageType('StatsRequest', (_message.Message,), { - 'DESCRIPTOR' : _STATSREQUEST, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.StatsRequest) - }) -_sym_db.RegisterMessage(StatsRequest) - -StatsResponse = _reflection.GeneratedProtocolMessageType('StatsResponse', (_message.Message,), { - 'DESCRIPTOR' : _STATSRESPONSE, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.StatsResponse) - }) -_sym_db.RegisterMessage(StatsResponse) - -_TRAVERSALSERVICE = DESCRIPTOR.services_by_name['TraversalService'] -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\036org.softwareheritage.graph.rpcB\014GraphServiceP\001' - _GRAPHDIRECTION._serialized_start=2853 - _GRAPHDIRECTION._serialized_end=2896 - _GETNODEREQUEST._serialized_start=77 - _GETNODEREQUEST._serialized_end=164 - _TRAVERSALREQUEST._serialized_start=167 - _TRAVERSALREQUEST._serialized_end=511 - _FINDPATHTOREQUEST._serialized_start=514 - _FINDPATHTOREQUEST._serialized_end=793 - _FINDPATHBETWEENREQUEST._serialized_start=796 - _FINDPATHBETWEENREQUEST._serialized_end=1181 - _NODEFILTER._serialized_start=1184 - _NODEFILTER._serialized_end=1362 - _NODE._serialized_start=1365 - _NODE._serialized_end=1639 - _PATH._serialized_start=1641 - _PATH._serialized_end=1726 - _SUCCESSOR._serialized_start=1728 - _SUCCESSOR._serialized_end=1806 - _CONTENTDATA._serialized_start=1808 - _CONTENTDATA._serialized_end=1893 - _REVISIONDATA._serialized_start=1896 - _REVISIONDATA._serialized_end=2222 - _RELEASEDATA._serialized_start=2225 - _RELEASEDATA._serialized_end=2430 - _ORIGINDATA._serialized_start=2432 - _ORIGINDATA._serialized_end=2470 - _EDGELABEL._serialized_start=2472 - _EDGELABEL._serialized_end=2517 - _COUNTRESPONSE._serialized_start=2519 - _COUNTRESPONSE._serialized_end=2549 - _STATSREQUEST._serialized_start=2551 - _STATSREQUEST._serialized_end=2565 - _STATSRESPONSE._serialized_start=2568 - _STATSRESPONSE._serialized_end=2851 - _TRAVERSALSERVICE._serialized_start=2899 - _TRAVERSALSERVICE._serialized_end=3362 -# @@protoc_insertion_point(module_scope) diff --git a/swh/graph/tests/test_grpc.py b/swh/graph/tests/test_grpc.py index 2cef192..a98c549 100644 --- a/swh/graph/tests/test_grpc.py +++ b/swh/graph/tests/test_grpc.py @@ -1,129 +1,129 @@ # Copyright (c) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import hashlib from google.protobuf.field_mask_pb2 import FieldMask -from swh.graph.rpc.swhgraph_pb2 import ( +from swh.graph.grpc.swhgraph_pb2 import ( GraphDirection, NodeFilter, StatsRequest, TraversalRequest, ) TEST_ORIGIN_ID = "swh:1:ori:{}".format( hashlib.sha1(b"https://example.com/swh/graph").hexdigest() ) def test_stats(graph_grpc_stub): stats = graph_grpc_stub.Stats(StatsRequest()) assert stats.num_nodes == 21 assert stats.num_edges == 23 assert isinstance(stats.compression_ratio, float) assert isinstance(stats.bits_per_node, float) assert isinstance(stats.bits_per_edge, float) assert isinstance(stats.avg_locality, float) assert stats.indegree_min == 0 assert stats.indegree_max == 3 assert isinstance(stats.indegree_avg, float) assert stats.outdegree_min == 0 assert stats.outdegree_max == 3 assert isinstance(stats.outdegree_avg, float) def test_leaves(graph_grpc_stub): request = graph_grpc_stub.Traverse( TraversalRequest( src=[TEST_ORIGIN_ID], mask=FieldMask(paths=["swhid"]), return_nodes=NodeFilter(types="cnt"), ) ) actual = [node.swhid for node in request] expected = [ "swh:1:cnt:0000000000000000000000000000000000000001", "swh:1:cnt:0000000000000000000000000000000000000004", "swh:1:cnt:0000000000000000000000000000000000000005", "swh:1:cnt:0000000000000000000000000000000000000007", ] assert set(actual) == set(expected) def test_neighbors(graph_grpc_stub): request = graph_grpc_stub.Traverse( TraversalRequest( src=["swh:1:rev:0000000000000000000000000000000000000009"], direction=GraphDirection.BACKWARD, mask=FieldMask(paths=["swhid"]), min_depth=1, max_depth=1, ) ) actual = [node.swhid for node in request] expected = [ "swh:1:snp:0000000000000000000000000000000000000020", "swh:1:rel:0000000000000000000000000000000000000010", "swh:1:rev:0000000000000000000000000000000000000013", ] assert set(actual) == set(expected) def test_visit_nodes(graph_grpc_stub): request = graph_grpc_stub.Traverse( TraversalRequest( src=["swh:1:rel:0000000000000000000000000000000000000010"], mask=FieldMask(paths=["swhid"]), edges="rel:rev,rev:rev", ) ) actual = [node.swhid for node in request] expected = [ "swh:1:rel:0000000000000000000000000000000000000010", "swh:1:rev:0000000000000000000000000000000000000009", "swh:1:rev:0000000000000000000000000000000000000003", ] assert set(actual) == set(expected) def test_visit_nodes_filtered(graph_grpc_stub): request = graph_grpc_stub.Traverse( TraversalRequest( src=["swh:1:rel:0000000000000000000000000000000000000010"], mask=FieldMask(paths=["swhid"]), return_nodes=NodeFilter(types="dir"), ) ) actual = [node.swhid for node in request] expected = [ "swh:1:dir:0000000000000000000000000000000000000002", "swh:1:dir:0000000000000000000000000000000000000008", "swh:1:dir:0000000000000000000000000000000000000006", ] assert set(actual) == set(expected) def test_visit_nodes_filtered_star(graph_grpc_stub): request = graph_grpc_stub.Traverse( TraversalRequest( src=["swh:1:rel:0000000000000000000000000000000000000010"], mask=FieldMask(paths=["swhid"]), ) ) actual = [node.swhid for node in request] expected = [ "swh:1:rel:0000000000000000000000000000000000000010", "swh:1:rev:0000000000000000000000000000000000000009", "swh:1:rev:0000000000000000000000000000000000000003", "swh:1:dir:0000000000000000000000000000000000000002", "swh:1:cnt:0000000000000000000000000000000000000001", "swh:1:dir:0000000000000000000000000000000000000008", "swh:1:cnt:0000000000000000000000000000000000000007", "swh:1:dir:0000000000000000000000000000000000000006", "swh:1:cnt:0000000000000000000000000000000000000004", "swh:1:cnt:0000000000000000000000000000000000000005", ] assert set(actual) == set(expected)