Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9311866
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
151 KB
Subscribers
None
View Options
diff --git a/docs/grpc-api.rst b/docs/grpc-api.rst
new file mode 100644
index 0000000..d4e2b8c
--- /dev/null
+++ b/docs/grpc-api.rst
@@ -0,0 +1,11 @@
+.. _swh-graph-grpc-api:
+
+Using the GRPC API
+==================
+
+Protobuf API Specification
+--------------------------
+
+.. literalinclude:: ../proto/swhgraph.proto
+ :language: protobuf
+ :linenos:
diff --git a/docs/index.rst b/docs/index.rst
index ec4b098..07e1068 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -1,18 +1,19 @@
.. _swh-graph:
.. include:: README.rst
.. toctree::
:maxdepth: 1
:caption: Overview
quickstart
api
+ grpc-api
java-api
memory
compression
cli
docker
git2graph
/apidoc/swh.graph
diff --git a/java/src/main/java/org/softwareheritage/graph/rpc/GraphServer.java b/java/src/main/java/org/softwareheritage/graph/rpc/GraphServer.java
index 3719ec5..3fa4359 100644
--- a/java/src/main/java/org/softwareheritage/graph/rpc/GraphServer.java
+++ b/java/src/main/java/org/softwareheritage/graph/rpc/GraphServer.java
@@ -1,274 +1,274 @@
package org.softwareheritage.graph.rpc;
import com.google.protobuf.FieldMask;
import com.martiansoftware.jsap.*;
import io.grpc.Server;
import io.grpc.Status;
import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder;
import io.grpc.netty.shaded.io.netty.channel.ChannelOption;
import io.grpc.stub.StreamObserver;
import io.grpc.protobuf.services.ProtoReflectionService;
import it.unimi.dsi.logging.ProgressLogger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.softwareheritage.graph.SWHID;
import org.softwareheritage.graph.SwhBidirectionalGraph;
import org.softwareheritage.graph.compress.LabelMapBuilder;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Properties;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
/**
* Server that manages startup/shutdown of a {@code Greeter} server.
*/
public class GraphServer {
private final static Logger logger = LoggerFactory.getLogger(GraphServer.class);
private final SwhBidirectionalGraph graph;
private final int port;
private final int threads;
private Server server;
public GraphServer(String graphBasename, int port, int threads) throws IOException {
this.graph = loadGraph(graphBasename);
this.port = port;
this.threads = threads;
}
public static SwhBidirectionalGraph loadGraph(String basename) throws IOException {
// TODO: use loadLabelledMapped() when https://github.com/vigna/webgraph-big/pull/5 is merged
SwhBidirectionalGraph g = SwhBidirectionalGraph.loadLabelled(basename, new ProgressLogger(logger));
g.loadContentLength();
g.loadContentIsSkipped();
g.loadPersonIds();
g.loadAuthorTimestamps();
g.loadCommitterTimestamps();
g.loadMessages();
g.loadTagNames();
g.loadLabelNames();
return g;
}
private void start() throws IOException {
server = NettyServerBuilder.forPort(port).withChildOption(ChannelOption.SO_REUSEADDR, true)
.executor(Executors.newFixedThreadPool(threads)).addService(new TraversalService(graph))
.addService(ProtoReflectionService.newInstance()).build().start();
logger.info("Server started, listening on " + port);
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
GraphServer.this.stop();
} catch (InterruptedException e) {
e.printStackTrace(System.err);
}
}));
}
private void stop() throws InterruptedException {
if (server != null) {
server.shutdown().awaitTermination(30, TimeUnit.SECONDS);
}
}
/**
* Await termination on the main thread since the grpc library uses daemon threads.
*/
private void blockUntilShutdown() throws InterruptedException {
if (server != null) {
server.awaitTermination();
}
}
private static JSAPResult parseArgs(String[] args) {
JSAPResult config = null;
try {
SimpleJSAP jsap = new SimpleJSAP(LabelMapBuilder.class.getName(), "",
new Parameter[]{
new FlaggedOption("port", JSAP.INTEGER_PARSER, "50091", JSAP.NOT_REQUIRED, 'p', "port",
"The port on which the server should listen."),
new FlaggedOption("threads", JSAP.INTEGER_PARSER, "1", JSAP.NOT_REQUIRED, 't', "threads",
"The number of concurrent threads. 0 = number of cores."),
new UnflaggedOption("graphBasename", JSAP.STRING_PARSER, JSAP.REQUIRED,
"Basename of the output graph")});
config = jsap.parse(args);
if (jsap.messagePrinted()) {
System.exit(1);
}
} catch (JSAPException e) {
e.printStackTrace();
}
return config;
}
/**
* Main launches the server from the command line.
*/
public static void main(String[] args) throws IOException, InterruptedException {
JSAPResult config = parseArgs(args);
String graphBasename = config.getString("graphBasename");
int port = config.getInt("port");
int threads = config.getInt("threads");
if (threads == 0) {
threads = Runtime.getRuntime().availableProcessors();
}
final GraphServer server = new GraphServer(graphBasename, port, threads);
server.start();
server.blockUntilShutdown();
}
static class TraversalService extends TraversalServiceGrpc.TraversalServiceImplBase {
SwhBidirectionalGraph graph;
public TraversalService(SwhBidirectionalGraph graph) {
this.graph = graph;
}
@Override
public void stats(StatsRequest request, StreamObserver<StatsResponse> responseObserver) {
StatsResponse.Builder response = StatsResponse.newBuilder();
response.setNumNodes(graph.numNodes());
response.setNumEdges(graph.numArcs());
Properties properties = new Properties();
try {
properties.load(new FileInputStream(graph.getPath() + ".properties"));
properties.load(new FileInputStream(graph.getPath() + ".stats"));
} catch (IOException e) {
throw new RuntimeException(e);
}
- response.setCompression(Double.parseDouble(properties.getProperty("compratio")));
+ response.setCompressionRatio(Double.parseDouble(properties.getProperty("compratio")));
response.setBitsPerNode(Double.parseDouble(properties.getProperty("bitspernode")));
response.setBitsPerEdge(Double.parseDouble(properties.getProperty("bitsperlink")));
response.setAvgLocality(Double.parseDouble(properties.getProperty("avglocality")));
response.setIndegreeMin(Long.parseLong(properties.getProperty("minindegree")));
response.setIndegreeMax(Long.parseLong(properties.getProperty("maxindegree")));
response.setIndegreeAvg(Double.parseDouble(properties.getProperty("avgindegree")));
response.setOutdegreeMin(Long.parseLong(properties.getProperty("minoutdegree")));
response.setOutdegreeMax(Long.parseLong(properties.getProperty("maxoutdegree")));
response.setOutdegreeAvg(Double.parseDouble(properties.getProperty("avgoutdegree")));
responseObserver.onNext(response.build());
responseObserver.onCompleted();
}
@Override
public void getNode(GetNodeRequest request, StreamObserver<Node> responseObserver) {
long nodeId;
try {
nodeId = graph.getNodeId(new SWHID(request.getSwhid()));
} catch (IllegalArgumentException e) {
responseObserver
.onError(Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e).asException());
return;
}
Node.Builder builder = Node.newBuilder();
NodePropertyBuilder.buildNodeProperties(graph.getForwardGraph(),
request.hasMask() ? request.getMask() : null, builder, nodeId);
responseObserver.onNext(builder.build());
responseObserver.onCompleted();
}
@Override
public void traverse(TraversalRequest request, StreamObserver<Node> responseObserver) {
SwhBidirectionalGraph g = graph.copy();
Traversal.SimpleTraversal t;
try {
t = new Traversal.SimpleTraversal(g, request, responseObserver::onNext);
} catch (IllegalArgumentException e) {
responseObserver
.onError(Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e).asException());
return;
}
t.visit();
responseObserver.onCompleted();
}
@Override
public void findPathTo(FindPathToRequest request, StreamObserver<Path> responseObserver) {
SwhBidirectionalGraph g = graph.copy();
Traversal.FindPathTo t;
try {
t = new Traversal.FindPathTo(g, request);
} catch (IllegalArgumentException e) {
responseObserver
.onError(Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e).asException());
return;
}
t.visit();
Path path = t.getPath();
if (path == null) {
responseObserver.onError(Status.NOT_FOUND.asException());
} else {
responseObserver.onNext(path);
responseObserver.onCompleted();
}
}
@Override
public void findPathBetween(FindPathBetweenRequest request, StreamObserver<Path> responseObserver) {
SwhBidirectionalGraph g = graph.copy();
Traversal.FindPathBetween t;
try {
t = new Traversal.FindPathBetween(g, request);
} catch (IllegalArgumentException e) {
responseObserver
.onError(Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e).asException());
return;
}
t.visit();
Path path = t.getPath();
if (path == null) {
responseObserver.onError(Status.NOT_FOUND.asException());
} else {
responseObserver.onNext(path);
responseObserver.onCompleted();
}
}
@Override
public void countNodes(TraversalRequest request, StreamObserver<CountResponse> responseObserver) {
AtomicLong count = new AtomicLong(0);
SwhBidirectionalGraph g = graph.copy();
TraversalRequest fixedReq = TraversalRequest.newBuilder(request)
// Ignore return fields, just count nodes
.setMask(FieldMask.getDefaultInstance()).build();
Traversal.SimpleTraversal t;
try {
t = new Traversal.SimpleTraversal(g, fixedReq, n -> count.incrementAndGet());
} catch (IllegalArgumentException e) {
responseObserver
.onError(Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e).asException());
return;
}
t.visit();
CountResponse response = CountResponse.newBuilder().setCount(count.get()).build();
responseObserver.onNext(response);
responseObserver.onCompleted();
}
@Override
public void countEdges(TraversalRequest request, StreamObserver<CountResponse> responseObserver) {
AtomicLong count = new AtomicLong(0);
SwhBidirectionalGraph g = graph.copy();
TraversalRequest fixedReq = TraversalRequest.newBuilder(request)
// Force return empty successors to count the edges
.setMask(FieldMask.newBuilder().addPaths("num_successors").build()).build();
Traversal.SimpleTraversal t;
try {
t = new Traversal.SimpleTraversal(g, fixedReq, n -> count.addAndGet(n.getNumSuccessors()));
} catch (IllegalArgumentException e) {
responseObserver
.onError(Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e).asException());
return;
}
t.visit();
CountResponse response = CountResponse.newBuilder().setCount(count.get()).build();
responseObserver.onNext(response);
responseObserver.onCompleted();
}
}
}
diff --git a/java/src/main/java/org/softwareheritage/graph/rpc/Traversal.java b/java/src/main/java/org/softwareheritage/graph/rpc/Traversal.java
index 553f603..ba8ed40 100644
--- a/java/src/main/java/org/softwareheritage/graph/rpc/Traversal.java
+++ b/java/src/main/java/org/softwareheritage/graph/rpc/Traversal.java
@@ -1,429 +1,429 @@
package org.softwareheritage.graph.rpc;
import it.unimi.dsi.big.webgraph.labelling.ArcLabelledNodeIterator;
import it.unimi.dsi.big.webgraph.labelling.Label;
import org.softwareheritage.graph.*;
import java.util.*;
public class Traversal {
private static ArcLabelledNodeIterator.LabelledArcIterator filterLabelledSuccessors(SwhUnidirectionalGraph g,
long nodeId, AllowedEdges allowedEdges) {
if (allowedEdges.restrictedTo == null) {
// All edges are allowed, bypass edge check
return g.labelledSuccessors(nodeId);
} else {
ArcLabelledNodeIterator.LabelledArcIterator allSuccessors = g.labelledSuccessors(nodeId);
return new ArcLabelledNodeIterator.LabelledArcIterator() {
@Override
public Label label() {
return allSuccessors.label();
}
@Override
public long nextLong() {
long neighbor;
while ((neighbor = allSuccessors.nextLong()) != -1) {
if (allowedEdges.isAllowed(g.getNodeType(nodeId), g.getNodeType(neighbor))) {
return neighbor;
}
}
return -1;
}
@Override
public long skip(final long n) {
long i = 0;
while (i < n && nextLong() != -1)
i++;
return i;
}
};
}
}
private static class NodeFilterChecker {
private final SwhUnidirectionalGraph g;
private final NodeFilter filter;
private final AllowedNodes allowedNodes;
private NodeFilterChecker(SwhUnidirectionalGraph graph, NodeFilter filter) {
this.g = graph;
this.filter = filter;
this.allowedNodes = new AllowedNodes(filter.hasTypes() ? filter.getTypes() : "*");
}
public boolean allowed(long nodeId) {
if (filter == null) {
return true;
}
if (!this.allowedNodes.isAllowed(g.getNodeType(nodeId))) {
return false;
}
return true;
}
}
public static SwhUnidirectionalGraph getDirectedGraph(SwhBidirectionalGraph g, GraphDirection direction) {
switch (direction) {
case FORWARD:
return g.getForwardGraph();
case BACKWARD:
return g.getBackwardGraph();
/*
* TODO: add support for BOTH case BOTH: return new SwhUnidirectionalGraph(g.symmetrize(),
* g.getProperties());
*/
default :
throw new IllegalArgumentException("Unknown direction: " + direction);
}
}
public static GraphDirection reverseDirection(GraphDirection direction) {
switch (direction) {
case FORWARD:
return GraphDirection.BACKWARD;
case BACKWARD:
return GraphDirection.FORWARD;
/*
* TODO: add support for BOTH case BOTH: return GraphDirection.BOTH;
*/
default :
throw new IllegalArgumentException("Unknown direction: " + direction);
}
}
static class StopTraversalException extends RuntimeException {
}
static class BFSVisitor {
protected final SwhUnidirectionalGraph g;
protected long depth = 0;
protected long traversalSuccessors = 0;
protected long edgesAccessed = 0;
protected HashMap<Long, Long> parents = new HashMap<>();
protected ArrayDeque<Long> queue = new ArrayDeque<>();
private long maxDepth = -1;
private long maxEdges = -1;
BFSVisitor(SwhUnidirectionalGraph g) {
this.g = g;
}
public void addSource(long nodeId) {
queue.add(nodeId);
parents.put(nodeId, -1L);
}
public void setMaxDepth(long depth) {
maxDepth = depth;
}
public void setMaxEdges(long edges) {
maxEdges = edges;
}
public void visitSetup() {
edgesAccessed = 0;
depth = 0;
queue.add(-1L); // depth sentinel
}
public void visit() {
visitSetup();
while (!queue.isEmpty()) {
visitStep();
}
}
public void visitStep() {
try {
assert !queue.isEmpty();
long curr = queue.poll();
if (curr == -1L) {
++depth;
if (!queue.isEmpty()) {
queue.add(-1L);
visitStep();
}
return;
}
if (maxDepth >= 0 && depth > maxDepth) {
throw new StopTraversalException();
}
edgesAccessed += g.outdegree(curr);
if (maxEdges >= 0 && edgesAccessed > maxEdges) {
throw new StopTraversalException();
}
visitNode(curr);
} catch (StopTraversalException e) {
// Traversal is over, clear the to-do queue.
queue.clear();
}
}
protected ArcLabelledNodeIterator.LabelledArcIterator getSuccessors(long nodeId) {
return g.labelledSuccessors(nodeId);
}
protected void visitNode(long node) {
ArcLabelledNodeIterator.LabelledArcIterator it = getSuccessors(node);
traversalSuccessors = 0;
for (long succ; (succ = it.nextLong()) != -1;) {
traversalSuccessors++;
visitEdge(node, succ, it.label());
}
}
protected void visitEdge(long src, long dst, Label label) {
if (!parents.containsKey(dst)) {
queue.add(dst);
parents.put(dst, src);
}
}
}
static class SimpleTraversal extends BFSVisitor {
private final NodeFilterChecker nodeReturnChecker;
private final AllowedEdges allowedEdges;
private final TraversalRequest request;
private final NodePropertyBuilder.NodeDataMask nodeDataMask;
private final NodeObserver nodeObserver;
private Node.Builder nodeBuilder;
SimpleTraversal(SwhBidirectionalGraph bidirectionalGraph, TraversalRequest request, NodeObserver nodeObserver) {
super(getDirectedGraph(bidirectionalGraph, request.getDirection()));
this.request = request;
this.nodeObserver = nodeObserver;
this.nodeReturnChecker = new NodeFilterChecker(g, request.getReturnNodes());
this.nodeDataMask = new NodePropertyBuilder.NodeDataMask(request.hasMask() ? request.getMask() : null);
this.allowedEdges = new AllowedEdges(request.hasEdges() ? request.getEdges() : "*");
request.getSrcList().forEach(srcSwhid -> {
long srcNodeId = g.getNodeId(new SWHID(srcSwhid));
addSource(srcNodeId);
});
if (request.hasMaxDepth()) {
setMaxDepth(request.getMaxDepth());
}
if (request.hasMaxEdges()) {
setMaxEdges(request.getMaxEdges());
}
}
@Override
protected ArcLabelledNodeIterator.LabelledArcIterator getSuccessors(long nodeId) {
return filterLabelledSuccessors(g, nodeId, allowedEdges);
}
@Override
public void visitNode(long node) {
nodeBuilder = null;
if (nodeReturnChecker.allowed(node) && (!request.hasMinDepth() || depth >= request.getMinDepth())) {
nodeBuilder = Node.newBuilder();
NodePropertyBuilder.buildNodeProperties(g, nodeDataMask, nodeBuilder, node);
}
super.visitNode(node);
if (request.getReturnNodes().hasMinTraversalSuccessors()
&& traversalSuccessors < request.getReturnNodes().getMinTraversalSuccessors()
|| request.getReturnNodes().hasMaxTraversalSuccessors()
&& traversalSuccessors > request.getReturnNodes().getMaxTraversalSuccessors()) {
nodeBuilder = null;
}
if (nodeBuilder != null) {
nodeObserver.onNext(nodeBuilder.build());
}
}
@Override
protected void visitEdge(long src, long dst, Label label) {
super.visitEdge(src, dst, label);
NodePropertyBuilder.buildSuccessorProperties(g, nodeDataMask, nodeBuilder, src, dst, label);
}
}
static class FindPathTo extends BFSVisitor {
private final AllowedEdges allowedEdges;
private final FindPathToRequest request;
private final NodePropertyBuilder.NodeDataMask nodeDataMask;
private final NodeFilterChecker targetChecker;
private Long targetNode = null;
FindPathTo(SwhBidirectionalGraph bidirectionalGraph, FindPathToRequest request) {
super(getDirectedGraph(bidirectionalGraph, request.getDirection()));
this.request = request;
this.targetChecker = new NodeFilterChecker(g, request.getTarget());
this.nodeDataMask = new NodePropertyBuilder.NodeDataMask(request.hasMask() ? request.getMask() : null);
this.allowedEdges = new AllowedEdges(request.hasEdges() ? request.getEdges() : "*");
if (request.hasMaxDepth()) {
setMaxDepth(request.getMaxDepth());
}
if (request.hasMaxEdges()) {
setMaxEdges(request.getMaxEdges());
}
request.getSrcList().forEach(srcSwhid -> {
long srcNodeId = g.getNodeId(new SWHID(srcSwhid));
addSource(srcNodeId);
});
}
@Override
protected ArcLabelledNodeIterator.LabelledArcIterator getSuccessors(long nodeId) {
return filterLabelledSuccessors(g, nodeId, allowedEdges);
}
@Override
public void visitNode(long node) {
if (targetChecker.allowed(node)) {
targetNode = node;
throw new StopTraversalException();
}
super.visitNode(node);
}
public Path getPath() {
if (targetNode == null) {
return null;
}
Path.Builder pathBuilder = Path.newBuilder();
long curNode = targetNode;
ArrayList<Long> path = new ArrayList<>();
while (curNode != -1) {
path.add(curNode);
curNode = parents.get(curNode);
}
Collections.reverse(path);
for (long nodeId : path) {
Node.Builder nodeBuilder = Node.newBuilder();
NodePropertyBuilder.buildNodeProperties(g, nodeDataMask, nodeBuilder, nodeId);
pathBuilder.addNode(nodeBuilder.build());
}
return pathBuilder.build();
}
}
static class FindPathBetween extends BFSVisitor {
private final FindPathBetweenRequest request;
private final NodePropertyBuilder.NodeDataMask nodeDataMask;
private final AllowedEdges allowedEdgesSrc;
private final AllowedEdges allowedEdgesDst;
private final BFSVisitor srcVisitor;
private final BFSVisitor dstVisitor;
private Long middleNode = null;
FindPathBetween(SwhBidirectionalGraph bidirectionalGraph, FindPathBetweenRequest request) {
super(getDirectedGraph(bidirectionalGraph, request.getDirection()));
this.request = request;
this.nodeDataMask = new NodePropertyBuilder.NodeDataMask(request.hasMask() ? request.getMask() : null);
GraphDirection direction = request.getDirection();
GraphDirection directionReverse = request.hasDirectionReverse()
? request.getDirectionReverse()
: reverseDirection(request.getDirection());
SwhUnidirectionalGraph srcGraph = getDirectedGraph(bidirectionalGraph, direction);
SwhUnidirectionalGraph dstGraph = getDirectedGraph(bidirectionalGraph, directionReverse);
this.allowedEdgesSrc = new AllowedEdges(request.hasEdges() ? request.getEdges() : "*");
this.allowedEdgesDst = request.hasEdgesReverse()
? new AllowedEdges(request.getEdgesReverse())
: (request.hasEdges()
? (direction == directionReverse
? new AllowedEdges(request.getEdges())
: new AllowedEdges(request.getEdges()).reverse())
: new AllowedEdges("*"));
this.srcVisitor = new BFSVisitor(srcGraph) {
@Override
protected ArcLabelledNodeIterator.LabelledArcIterator getSuccessors(long nodeId) {
return filterLabelledSuccessors(g, nodeId, allowedEdgesSrc);
}
@Override
public void visitNode(long node) {
if (dstVisitor.parents.containsKey(node)) {
middleNode = node;
throw new StopTraversalException();
}
super.visitNode(node);
}
};
this.dstVisitor = new BFSVisitor(dstGraph) {
@Override
protected ArcLabelledNodeIterator.LabelledArcIterator getSuccessors(long nodeId) {
return filterLabelledSuccessors(g, nodeId, allowedEdgesDst);
}
@Override
public void visitNode(long node) {
if (srcVisitor.parents.containsKey(node)) {
middleNode = node;
throw new StopTraversalException();
}
super.visitNode(node);
}
};
if (request.hasMaxDepth()) {
this.srcVisitor.setMaxDepth(request.getMaxDepth());
this.dstVisitor.setMaxDepth(request.getMaxDepth());
}
if (request.hasMaxEdges()) {
this.srcVisitor.setMaxEdges(request.getMaxEdges());
this.dstVisitor.setMaxEdges(request.getMaxEdges());
}
request.getSrcList().forEach(srcSwhid -> {
long srcNodeId = g.getNodeId(new SWHID(srcSwhid));
srcVisitor.addSource(srcNodeId);
});
request.getDstList().forEach(srcSwhid -> {
long srcNodeId = g.getNodeId(new SWHID(srcSwhid));
dstVisitor.addSource(srcNodeId);
});
}
@Override
public void visit() {
srcVisitor.visitSetup();
dstVisitor.visitSetup();
while (!srcVisitor.queue.isEmpty() || !dstVisitor.queue.isEmpty()) {
if (!srcVisitor.queue.isEmpty()) {
srcVisitor.visitStep();
}
if (!dstVisitor.queue.isEmpty()) {
dstVisitor.visitStep();
}
}
}
public Path getPath() {
if (middleNode == null) {
return null;
}
Path.Builder pathBuilder = Path.newBuilder();
ArrayList<Long> path = new ArrayList<>();
long curNode = middleNode;
while (curNode != -1) {
path.add(curNode);
curNode = srcVisitor.parents.get(curNode);
}
- pathBuilder.setMiddleNodeIndex(path.size() - 1);
+ pathBuilder.setMidpointIndex(path.size() - 1);
Collections.reverse(path);
curNode = dstVisitor.parents.get(middleNode);
while (curNode != -1) {
path.add(curNode);
curNode = dstVisitor.parents.get(curNode);
}
for (long nodeId : path) {
Node.Builder nodeBuilder = Node.newBuilder();
NodePropertyBuilder.buildNodeProperties(g, nodeDataMask, nodeBuilder, nodeId);
pathBuilder.addNode(nodeBuilder.build());
}
return pathBuilder.build();
}
}
public interface NodeObserver {
void onNext(Node nodeId);
}
}
diff --git a/java/src/test/java/org/softwareheritage/graph/rpc/FindPathBetweenTest.java b/java/src/test/java/org/softwareheritage/graph/rpc/FindPathBetweenTest.java
index 7a7b682..be76492 100644
--- a/java/src/test/java/org/softwareheritage/graph/rpc/FindPathBetweenTest.java
+++ b/java/src/test/java/org/softwareheritage/graph/rpc/FindPathBetweenTest.java
@@ -1,203 +1,203 @@
package org.softwareheritage.graph.rpc;
import io.grpc.Status;
import io.grpc.StatusRuntimeException;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.softwareheritage.graph.SWHID;
import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
public class FindPathBetweenTest extends TraversalServiceTest {
private FindPathBetweenRequest.Builder getRequestBuilder(SWHID src, SWHID dst) {
return FindPathBetweenRequest.newBuilder().addSrc(src.toString()).addDst(dst.toString());
}
@Test
public void testSwhidErrors() {
StatusRuntimeException thrown;
thrown = assertThrows(StatusRuntimeException.class, () -> client
.findPathBetween(FindPathBetweenRequest.newBuilder().addSrc(fakeSWHID("cnt", 404).toString()).build()));
assertEquals(Status.INVALID_ARGUMENT.getCode(), thrown.getStatus().getCode());
thrown = assertThrows(StatusRuntimeException.class, () -> client.findPathBetween(FindPathBetweenRequest
.newBuilder().addSrc("swh:1:lol:0000000000000000000000000000000000000001").build()));
assertEquals(Status.INVALID_ARGUMENT.getCode(), thrown.getStatus().getCode());
thrown = assertThrows(StatusRuntimeException.class, () -> client.findPathBetween(FindPathBetweenRequest
.newBuilder().addSrc("swh:1:cnt:000000000000000000000000000000000000000z").build()));
assertEquals(Status.INVALID_ARGUMENT.getCode(), thrown.getStatus().getCode());
thrown = assertThrows(StatusRuntimeException.class,
() -> client.findPathBetween(FindPathBetweenRequest.newBuilder().addSrc(TEST_ORIGIN_ID)
.addDst("swh:1:cnt:000000000000000000000000000000000000000z").build()));
assertEquals(Status.INVALID_ARGUMENT.getCode(), thrown.getStatus().getCode());
}
@Test
public void testEdgeErrors() {
StatusRuntimeException thrown;
thrown = assertThrows(StatusRuntimeException.class, () -> client.findPathBetween(FindPathBetweenRequest
.newBuilder().addSrc(TEST_ORIGIN_ID).addDst(TEST_ORIGIN_ID).setEdges("batracien:reptile").build()));
assertEquals(Status.INVALID_ARGUMENT.getCode(), thrown.getStatus().getCode());
}
// Test path between ori 1 and cnt 4 (forward graph)
@Test
public void forwardRootToLeaf() {
ArrayList<SWHID> actual = getSWHIDs(
client.findPathBetween(getRequestBuilder(new SWHID(TEST_ORIGIN_ID), fakeSWHID("cnt", 4)).build()));
List<SWHID> expected = List.of(new SWHID(TEST_ORIGIN_ID), fakeSWHID("snp", 20), fakeSWHID("rev", 9),
fakeSWHID("dir", 8), fakeSWHID("dir", 6), fakeSWHID("cnt", 4));
Assertions.assertEquals(expected, actual);
}
// Test path between rev 18 and rev 3 (forward graph)
@Test
public void forwardRevToRev() {
ArrayList<SWHID> actual = getSWHIDs(
client.findPathBetween(getRequestBuilder(fakeSWHID("rev", 18), fakeSWHID("rev", 3)).build()));
List<SWHID> expected = List.of(fakeSWHID("rev", 18), fakeSWHID("rev", 13), fakeSWHID("rev", 9),
fakeSWHID("rev", 3));
Assertions.assertEquals(expected, actual);
}
// Test path between rev 3 and rev 18 (backward graph)
@Test
public void backwardRevToRev() {
ArrayList<SWHID> actual = getSWHIDs(
client.findPathBetween(getRequestBuilder(fakeSWHID("rev", 3), fakeSWHID("rev", 18))
.setDirection(GraphDirection.BACKWARD).build()));
List<SWHID> expected = List.of(fakeSWHID("rev", 3), fakeSWHID("rev", 9), fakeSWHID("rev", 13),
fakeSWHID("rev", 18));
Assertions.assertEquals(expected, actual);
}
// Test path between cnt 4 and itself (forward graph)
@Test
public void forwardCntToItself() {
ArrayList<SWHID> actual = getSWHIDs(
client.findPathBetween(getRequestBuilder(fakeSWHID("cnt", 4), fakeSWHID("cnt", 4)).build()));
List<SWHID> expected = List.of(fakeSWHID("cnt", 4));
Assertions.assertEquals(expected, actual);
}
// Start from ori and rel 19 and find cnt 14 or cnt 7 (forward graph)
@Test
public void forwardMultipleSourcesDest() {
ArrayList<SWHID> actual = getSWHIDs(
client.findPathBetween(getRequestBuilder(fakeSWHID("rel", 19), fakeSWHID("cnt", 14))
.addSrc(TEST_ORIGIN_ID).addDst(fakeSWHID("cnt", 7).toString()).build()));
List<SWHID> expected = List.of(fakeSWHID("rel", 19), fakeSWHID("rev", 18), fakeSWHID("dir", 17),
fakeSWHID("cnt", 14));
}
// Start from cnt 4 and cnt 11 and find rev 13 or rev 9 (backward graph)
@Test
public void backwardMultipleSourcesDest() {
ArrayList<SWHID> actual = getSWHIDs(client.findPathBetween(
getRequestBuilder(fakeSWHID("cnt", 4), fakeSWHID("rev", 13)).setDirection(GraphDirection.BACKWARD)
.addSrc(fakeSWHID("cnt", 11).toString()).addDst(fakeSWHID("rev", 9).toString()).build()));
List<SWHID> expected = List.of(fakeSWHID("cnt", 11), fakeSWHID("dir", 12), fakeSWHID("rev", 13));
Assertions.assertEquals(expected, actual);
}
// Start from all directories and find the origin (backward graph)
@Test
public void backwardMultipleSourcesAllDirToOri() {
ArrayList<SWHID> actual = getSWHIDs(
client.findPathBetween(getRequestBuilder(fakeSWHID("dir", 2), new SWHID(TEST_ORIGIN_ID))
.addSrc(fakeSWHID("dir", 6).toString()).addSrc(fakeSWHID("dir", 8).toString())
.addSrc(fakeSWHID("dir", 12).toString()).addSrc(fakeSWHID("dir", 16).toString())
.addSrc(fakeSWHID("dir", 17).toString()).setDirection(GraphDirection.BACKWARD).build()));
List<SWHID> expected = List.of(fakeSWHID("dir", 8), fakeSWHID("rev", 9), fakeSWHID("snp", 20),
new SWHID(TEST_ORIGIN_ID));
Assertions.assertEquals(expected, actual);
}
// Start from cnt 4 and find any rev (backward graph)
@Test
public void backwardCntToAnyRev() {
ArrayList<SWHID> actual = getSWHIDs(
client.findPathBetween(getRequestBuilder(fakeSWHID("cnt", 4), fakeSWHID("rev", 3))
.addDst(fakeSWHID("rev", 9).toString()).addDst(fakeSWHID("rev", 13).toString())
.addDst(fakeSWHID("rev", 18).toString()).setDirection(GraphDirection.BACKWARD).build()));
List<SWHID> expected = List.of(fakeSWHID("cnt", 4), fakeSWHID("dir", 6), fakeSWHID("dir", 8),
fakeSWHID("rev", 9));
Assertions.assertEquals(expected, actual);
}
// Impossible path between rev 9 and cnt 14
@Test
public void forwardImpossiblePath() {
StatusRuntimeException thrown = Assertions.assertThrows(StatusRuntimeException.class, () -> {
client.findPathBetween(getRequestBuilder(fakeSWHID("rev", 9), fakeSWHID("cnt", 14)).build());
});
Assertions.assertEquals(thrown.getStatus().getCode(), Status.NOT_FOUND.getCode());
// Reverse direction
thrown = Assertions.assertThrows(StatusRuntimeException.class, () -> {
client.findPathBetween(getRequestBuilder(fakeSWHID("cnt", 14), fakeSWHID("rev", 9))
.setDirection(GraphDirection.BACKWARD).build());
});
Assertions.assertEquals(thrown.getStatus().getCode(), Status.NOT_FOUND.getCode());
}
// Common ancestor between cnt 4 and cnt 15 : rev 18
@Test
public void commonAncestorBackwardBackward() {
Path p = client.findPathBetween(getRequestBuilder(fakeSWHID("cnt", 4), fakeSWHID("cnt", 15))
.setDirection(GraphDirection.BACKWARD).setDirectionReverse(GraphDirection.BACKWARD).build());
ArrayList<SWHID> actual = getSWHIDs(p);
SWHID expected = fakeSWHID("rev", 18);
- Assertions.assertEquals(expected, actual.get(p.getMiddleNodeIndex()));
+ Assertions.assertEquals(expected, actual.get(p.getMidpointIndex()));
}
// Common descendant between rev 13 and rev 3 : cnt 1 (with rev:dir,dir:dir,dir:cnt)
@Test
public void commonDescendantForwardForward() {
Path p = client.findPathBetween(
getRequestBuilder(fakeSWHID("rev", 13), fakeSWHID("rev", 3)).setDirection(GraphDirection.FORWARD)
.setDirectionReverse(GraphDirection.FORWARD).setEdges("rev:dir,dir:dir,dir:cnt").build());
ArrayList<SWHID> actual = getSWHIDs(p);
SWHID expected = fakeSWHID("cnt", 1);
- Assertions.assertEquals(expected, actual.get(p.getMiddleNodeIndex()));
+ Assertions.assertEquals(expected, actual.get(p.getMidpointIndex()));
}
// Path between rel 19 and cnt 15 with various max depths
@Test
public void maxDepth() {
// Works with max_depth = 2
ArrayList<SWHID> actual = getSWHIDs(client
.findPathBetween(getRequestBuilder(fakeSWHID("rel", 19), fakeSWHID("cnt", 15)).setMaxDepth(2).build()));
List<SWHID> expected = List.of(fakeSWHID("rel", 19), fakeSWHID("rev", 18), fakeSWHID("dir", 17),
fakeSWHID("dir", 16), fakeSWHID("cnt", 15));
Assertions.assertEquals(expected, actual);
// Check that it throws NOT_FOUND with max depth = 1
StatusRuntimeException thrown = Assertions.assertThrows(StatusRuntimeException.class, () -> {
client.findPathBetween(
getRequestBuilder(fakeSWHID("rel", 19), fakeSWHID("cnt", 15)).setMaxDepth(1).build());
});
Assertions.assertEquals(thrown.getStatus().getCode(), Status.NOT_FOUND.getCode());
}
// Path between rel 19 and cnt 15 with various max edges
@Test
public void maxEdges() {
// Works with max_edges = 3
ArrayList<SWHID> actual = getSWHIDs(client
.findPathBetween(getRequestBuilder(fakeSWHID("rel", 19), fakeSWHID("cnt", 15)).setMaxEdges(3).build()));
List<SWHID> expected = List.of(fakeSWHID("rel", 19), fakeSWHID("rev", 18), fakeSWHID("dir", 17),
fakeSWHID("dir", 16), fakeSWHID("cnt", 15));
Assertions.assertEquals(expected, actual);
// Check that it throws NOT_FOUND with max_edges = 2
StatusRuntimeException thrown = Assertions.assertThrows(StatusRuntimeException.class, () -> {
client.findPathBetween(
getRequestBuilder(fakeSWHID("rel", 19), fakeSWHID("cnt", 15)).setMaxEdges(2).build());
});
Assertions.assertEquals(thrown.getStatus().getCode(), Status.NOT_FOUND.getCode());
}
}
diff --git a/proto/swhgraph.proto b/proto/swhgraph.proto
index db418d8..ca3c897 100644
--- a/proto/swhgraph.proto
+++ b/proto/swhgraph.proto
@@ -1,272 +1,316 @@
syntax = "proto3";
import "google/protobuf/field_mask.proto";
option java_multiple_files = true;
option java_package = "org.softwareheritage.graph.rpc";
option java_outer_classname = "GraphService";
package swh.graph;
+/* Graph traversal service */
service TraversalService {
/* GetNode returns a single Node and its properties. */
rpc GetNode (GetNodeRequest) returns (Node);
/* Traverse performs a breadth-first graph traversal from a set of source
- * nodes, then streams the nodes it encounters, along with their properties.
+ * nodes, then streams the nodes it encounters (if they match a given
+ * return filter), along with their properties.
*/
rpc Traverse (TraversalRequest) returns (stream Node);
/* FindPathTo searches for the shortest path between a set of source nodes
* and a node that matches a specific *criteria*.
+ *
+ * It does so by performing a breadth-first search from the source node,
+ * until any node that matches the given criteria is found, then follows
+ * back its parents to return the shortest path from the source set to that
+ * node.
*/
rpc FindPathTo (FindPathToRequest) returns (Path);
/* FindPathBetween searches for the shortest path between a set of source
- * nodes and a set of destination nodes. */
+ * nodes and a set of destination nodes.
+ *
+ * It does so by performing a *bidirectional breadth-first search*, i.e.,
+ * two parallel breadth-first searches, one from the source set ("src-BFS")
+ * and one from the destination set ("dst-BFS"), until both searches find a
+ * common node that joins their visited sets. This node is called the
+ * "midpoint node".
+ * The path returned is the path src -> ... -> midpoint * -> ... -> dst,
+ * which is the shortest path between src and dst.
+ *
+ * The graph direction of both BFS can be configured separately. By
+ * default, the dst-BFS will use the graph in the opposite direction than
+ * the src-BFS (if direction = FORWARD, by default direction_reverse =
+ * BACKWARD, and vice-versa). The default behavior is thus to search for
+ * the shortest path between two nodes in a given direction. However, one
+ * can also specify FORWARD or BACKWARD for *both* the src-BFS and the
+ * dst-BFS. This will search for a common descendant or a common ancestor
+ * between the two sets, respectively. These will be the midpoints of the
+ * returned path.
+ */
rpc FindPathBetween (FindPathBetweenRequest) returns (Path);
/* CountNodes does the same as Traverse, but only returns the number of
* nodes accessed during the traversal. */
rpc CountNodes (TraversalRequest) returns (CountResponse);
/* CountEdges does the same as Traverse, but only returns the number of
* edges accessed during the traversal. */
rpc CountEdges (TraversalRequest) returns (CountResponse);
/* Stats returns various */
rpc Stats (StatsRequest) returns (StatsResponse);
}
+/* Direction of the graph */
enum GraphDirection {
+ /* Forward DAG: ori -> snp -> rel -> rev -> dir -> cnt */
FORWARD = 0;
+ /* Transposed DAG: cnt -> dir -> rev -> rel -> snp -> ori */
BACKWARD = 1;
}
/* Describe a node to return */
message GetNodeRequest {
/* SWHID of the node to return */
string swhid = 1;
/* FieldMask of which fields are to be returned (e.g., "swhid,cnt.length").
* By default, all fields are returned. */
optional google.protobuf.FieldMask mask = 8;
}
/* TraversalRequest describes how a breadth-first traversal should be
* performed, and what should be returned to the client. */
message TraversalRequest {
/* Set of source nodes (SWHIDs) */
repeated string src = 1;
/* Direction of the graph to traverse. Defaults to FORWARD. */
GraphDirection direction = 2;
/* Edge restriction string (e.g. "rev:dir,dir:cnt").
* Defaults to "*" (all). */
optional string edges = 3;
/* Maximum number of edges accessed in the traversal, after which it stops.
* Defaults to infinite. */
optional int64 max_edges = 4;
/* Do not return nodes with a depth lower than this number.
* By default, all depths are returned. */
optional int64 min_depth = 5;
/* Maximum depth of the traversal, after which it stops.
* Defaults to infinite. */
optional int64 max_depth = 6;
/* Filter which nodes will be sent to the stream. By default, all nodes are
* returned. */
optional NodeFilter return_nodes = 7;
/* FieldMask of which fields are to be returned (e.g., "swhid,cnt.length").
* By default, all fields are returned. */
optional google.protobuf.FieldMask mask = 8;
}
/* FindPathToRequest describes a request to find the shortest path between a
* set of nodes and a given target criteria, as well as what should be returned
* in the path.
*/
message FindPathToRequest {
/* Set of source nodes (SWHIDs) */
repeated string src = 1;
/* Target criteria, i.e., what constitutes a valid path destination. */
NodeFilter target = 2;
/* Direction of the graph to traverse. Defaults to FORWARD. */
GraphDirection direction = 3;
/* Edge restriction string (e.g. "rev:dir,dir:cnt").
* Defaults to "*" (all). */
optional string edges = 4;
/* Maximum number of edges accessed in the traversal, after which it stops.
* Defaults to infinite. */
optional int64 max_edges = 5;
/* Maximum depth of the traversal, after which it stops.
* Defaults to infinite. */
optional int64 max_depth = 6;
/* FieldMask of which fields are to be returned (e.g., "swhid,cnt.length").
* By default, all fields are returned. */
optional google.protobuf.FieldMask mask = 7;
}
/* FindPathToRequest describes a request to find the shortest path between a
* set of source nodes and a set of destination nodes. It works by performing a
* bidirectional breadth-first traversal from both sets at the same time.
*/
message FindPathBetweenRequest {
/* Set of source nodes (SWHIDs) */
repeated string src = 1;
/* Set of destination nodes (SWHIDs) */
repeated string dst = 2;
/* Direction of the graph to traverse from the source set. Defaults to
* FORWARD. */
GraphDirection direction = 3;
/* Direction of the graph to traverse from the destination set. Defaults to
* the opposite of `direction`. If direction and direction_reverse are
* identical, it will find the first common successor of both sets in the
* given direction. */
optional GraphDirection direction_reverse = 4;
/* Edge restriction string for the traversal from the source set.
* (e.g. "rev:dir,dir:cnt"). Defaults to "*" (all). */
optional string edges = 5;
/* Edge restriction string for the reverse traversal from the destination
* set.
* If not specified:
* - If `edges` is not specified either, defaults to "*"
* - If direction == direction_reverse, defaults to `edges`
* - If direction != direction_reverse, defaults to the reverse of `edges`
* (e.g. "rev:dir" becomes "dir:rev").
*/
optional string edges_reverse = 6;
/* Maximum number of edges accessed in the traversal, after which it stops.
* Defaults to infinite. */
optional int64 max_edges = 7;
/* Maximum depth of the traversal, after which it stops.
* Defaults to infinite. */
optional int64 max_depth = 8;
/* FieldMask of which fields are to be returned (e.g., "swhid,cnt.length").
* By default, all fields are returned. */
optional google.protobuf.FieldMask mask = 9;
}
/* Represents various criteria that make a given node is "valid". A node is
* only valid if all the subcriteria present in this message are fulfilled.
*/
message NodeFilter {
/* Node restriction string. (e.g. "dir,cnt,rev"). Defaults to "*" (all). */
optional string types = 1;
/* Minimum number of successors encountered *during the traversal*.
* Default: no constraint */
optional int64 min_traversal_successors = 2;
/* Maximum number of successors encountered *during the traversal*.
* Default: no constraint */
optional int64 max_traversal_successors = 3;
}
/* Represents a node in the graph. */
message Node {
/* The SWHID of the graph node. */
string swhid = 1;
/* List of relevant successors of this node. */
repeated Successor successor = 2;
/* Number of relevant successors. */
optional int64 num_successors = 9;
/* Node properties */
oneof data {
ContentData cnt = 3;
RevisionData rev = 5;
ReleaseData rel = 6;
OriginData ori = 8;
};
}
/* Represents a path in the graph. */
message Path {
/* List of nodes in the path, from source to destination */
repeated Node node = 1;
- /* Index of the "middle node" of the path. For paths obtained in
+ /* Index of the "midpoint" of the path. For paths obtained with
* bidirectional search queries, this is the node that joined the two
* sets together. When looking for a common ancestor between two nodes by
* performing a FindPathBetween search with two backward graphs, this will
* be the index of the common ancestor in the path. */
- optional int32 middle_node_index = 2;
+ optional int32 midpoint_index = 2;
}
/* Represents a successor of a given node. */
message Successor {
/* The SWHID of the successor */
optional string swhid = 1;
/* A list of edge labels for the given edge */
repeated EdgeLabel label = 2;
}
/* Content node properties */
message ContentData {
/* Length of the blob, in bytes */
optional int64 length = 1;
/* Whether the content was skipped during ingestion. */
optional bool is_skipped = 2;
}
/* Revision node properties */
message RevisionData {
/* Revision author ID (anonymized) */
optional int64 author = 1;
/* UNIX timestamp of the revision date (UTC) */
optional int64 author_date = 2;
/* Timezone of the revision author date as an offset from UTC */
optional int32 author_date_offset = 3;
/* Revision committer ID (anonymized) */
optional int64 committer = 4;
/* UNIX timestamp of the revision committer date (UTC) */
optional int64 committer_date = 5;
/* Timezone of the revision committer date as an offset from UTC */
optional int32 committer_date_offset = 6;
/* Revision message */
optional bytes message = 7;
}
/* Release node properties */
message ReleaseData {
+ /* Release author ID (anonymized) */
optional int64 author = 1;
/* UNIX timestamp of the release date (UTC) */
optional int64 author_date = 2;
/* Timezone of the release author date as an offset from UTC */
optional int32 author_date_offset = 3;
/* Release name */
optional bytes name = 4;
/* Release message */
optional bytes message = 5;
}
/* Origin node properties */
message OriginData {
/* URL of the origin */
optional string url = 1;
}
message EdgeLabel {
/* Directory entry name for directories, branch name for snapshots */
bytes name = 1;
/* Entry permission (only set for directories). */
int32 permission = 2;
}
message CountResponse {
int64 count = 1;
}
message StatsRequest {
}
message StatsResponse {
+ /* Number of nodes in the graph */
int64 num_nodes = 1;
+ /* Number of edges in the graph */
int64 num_edges = 2;
- double compression = 3;
+ /* Ratio between the graph size and the information-theoretical lower
+ * bound */
+ double compression_ratio = 3;
+ /* Number of bits per node (overall graph size in bits divided by the
+ * number of nodes) */
double bits_per_node = 4;
+ /* Number of bits per edge (overall graph size in bits divided by the
+ * number of arcs). */
double bits_per_edge = 5;
double avg_locality = 6;
+ /* Smallest indegree */
int64 indegree_min = 7;
+ /* Largest indegree */
int64 indegree_max = 8;
+ /* Average indegree */
double indegree_avg = 9;
+ /* Smallest outdegree */
int64 outdegree_min = 10;
+ /* Largest outdegree */
int64 outdegree_max = 11;
+ /* Average outdegree */
double outdegree_avg = 12;
}
diff --git a/swh/graph/http_naive_client.py b/swh/graph/http_naive_client.py
index c8bf729..a94efe8 100644
--- a/swh/graph/http_naive_client.py
+++ b/swh/graph/http_naive_client.py
@@ -1,395 +1,395 @@
# Copyright (C) 2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import functools
import inspect
import re
import statistics
from typing import (
Callable,
Dict,
Iterable,
Iterator,
List,
Optional,
Set,
Tuple,
TypeVar,
Union,
)
from swh.model.swhids import CoreSWHID, ExtendedSWHID, ValidationError
from .http_client import GraphArgumentException
_NODE_TYPES = "ori|snp|rel|rev|dir|cnt"
NODES_RE = re.compile(rf"(\*|{_NODE_TYPES})")
EDGES_RE = re.compile(rf"(\*|{_NODE_TYPES}):(\*|{_NODE_TYPES})")
T = TypeVar("T", bound=Callable)
SWHIDlike = Union[CoreSWHID, ExtendedSWHID, str]
def check_arguments(f: T) -> T:
"""Decorator for generic argument checking for methods of NaiveClient.
Checks ``src`` is a valid and known SWHID, and ``edges`` has the right format."""
signature = inspect.signature(f)
@functools.wraps(f)
def newf(*args, **kwargs):
__tracebackhide__ = True # for pytest
try:
bound_args = signature.bind(*args, **kwargs)
except TypeError as e:
# rethrow the exception from here so pytest doesn't flood the terminal
# with signature.bind's call stack.
raise TypeError(*e.args) from None
self = bound_args.arguments["self"]
src = bound_args.arguments.get("src")
if src:
self._check_swhid(src)
edges = bound_args.arguments.get("edges")
if edges:
if edges != "*" and not EDGES_RE.match(edges):
raise GraphArgumentException(f"invalid edge restriction: {edges}")
return_types = bound_args.arguments.get("return_types")
if return_types:
if not NODES_RE.match(return_types):
raise GraphArgumentException(
f"invalid return_types restriction: {return_types}"
)
return f(*args, **kwargs)
return newf # type: ignore
def filter_node_types(node_types: str, nodes: Iterable[str]) -> Iterator[str]:
if node_types == "*":
yield from nodes
else:
prefixes = tuple(f"swh:1:{type_}:" for type_ in node_types.split(","))
for node in nodes:
if node.startswith(prefixes):
yield node
class NaiveClient:
"""An alternative implementation of the graph server, written in
pure-python and meant for simulating it in other components' test cases;
constructed from a list of nodes and (directed) edges, both represented as
SWHIDs.
It is NOT meant to be efficient in any way; only to be a very simple
implementation that provides the same behavior.
>>> nodes = [
... "swh:1:rev:1111111111111111111111111111111111111111",
... "swh:1:rev:2222222222222222222222222222222222222222",
... "swh:1:rev:3333333333333333333333333333333333333333",
... ]
>>> edges = [
... (
... "swh:1:rev:1111111111111111111111111111111111111111",
... "swh:1:rev:2222222222222222222222222222222222222222",
... ),
... (
... "swh:1:rev:2222222222222222222222222222222222222222",
... "swh:1:rev:3333333333333333333333333333333333333333",
... ),
... ]
>>> c = NaiveClient(nodes=nodes, edges=edges)
>>> list(c.leaves("swh:1:rev:1111111111111111111111111111111111111111"))
['swh:1:rev:3333333333333333333333333333333333333333']
"""
def __init__(
self, *, nodes: List[SWHIDlike], edges: List[Tuple[SWHIDlike, SWHIDlike]]
):
self.graph = Graph(nodes, edges)
def _check_swhid(self, swhid):
try:
ExtendedSWHID.from_string(swhid)
except ValidationError as e:
raise GraphArgumentException(*e.args) from None
if swhid not in self.graph.nodes:
raise GraphArgumentException(f"SWHID not found: {swhid}")
def stats(self) -> Dict:
return {
"num_nodes": len(self.graph.nodes),
"num_edges": sum(map(len, self.graph.forward_edges.values())),
- "compression": 1.0,
+ "compression_ratio": 1.0,
"bits_per_edge": 100.0,
"bits_per_node": 100.0,
"avg_locality": 0.0,
"indegree_min": min(map(len, self.graph.backward_edges.values())),
"indegree_max": max(map(len, self.graph.backward_edges.values())),
"indegree_avg": statistics.mean(
map(len, self.graph.backward_edges.values())
),
"outdegree_min": min(map(len, self.graph.forward_edges.values())),
"outdegree_max": max(map(len, self.graph.forward_edges.values())),
"outdegree_avg": statistics.mean(
map(len, self.graph.forward_edges.values())
),
}
@check_arguments
def leaves(
self,
src: str,
edges: str = "*",
direction: str = "forward",
max_edges: int = 0,
return_types: str = "*",
) -> Iterator[str]:
# TODO: max_edges
yield from filter_node_types(
return_types,
[
node
for node in self.graph.get_subgraph(src, edges, direction)
if not self.graph.get_filtered_neighbors(node, edges, direction)
],
)
@check_arguments
def neighbors(
self,
src: str,
edges: str = "*",
direction: str = "forward",
max_edges: int = 0,
return_types: str = "*",
) -> Iterator[str]:
# TODO: max_edges
yield from filter_node_types(
return_types, self.graph.get_filtered_neighbors(src, edges, direction)
)
@check_arguments
def visit_nodes(
self,
src: str,
edges: str = "*",
direction: str = "forward",
max_edges: int = 0,
return_types: str = "*",
) -> Iterator[str]:
# TODO: max_edges
yield from filter_node_types(
return_types, self.graph.get_subgraph(src, edges, direction)
)
@check_arguments
def visit_edges(
self, src: str, edges: str = "*", direction: str = "forward", max_edges: int = 0
) -> Iterator[Tuple[str, str]]:
if max_edges == 0:
max_edges = None # type: ignore
else:
max_edges -= 1
yield from list(self.graph.iter_edges_dfs(direction, edges, src))[:max_edges]
@check_arguments
def visit_paths(
self, src: str, edges: str = "*", direction: str = "forward", max_edges: int = 0
) -> Iterator[List[str]]:
# TODO: max_edges
for path in self.graph.iter_paths_dfs(direction, edges, src):
if path[-1] in self.leaves(src, edges, direction):
yield list(path)
@check_arguments
def walk(
self,
src: str,
dst: str,
edges: str = "*",
traversal: str = "dfs",
direction: str = "forward",
limit: Optional[int] = None,
) -> Iterator[str]:
# TODO: implement algo="bfs"
# TODO: limit
match_path: Callable[[str], bool]
if ":" in dst:
match_path = dst.__eq__
self._check_swhid(dst)
else:
match_path = lambda node: node.startswith(f"swh:1:{dst}:") # noqa
for path in self.graph.iter_paths_dfs(direction, edges, src):
if match_path(path[-1]):
if not limit:
# 0 or None
yield from path
elif limit > 0:
yield from path[0:limit]
else:
yield from path[limit:]
@check_arguments
def random_walk(
self,
src: str,
dst: str,
edges: str = "*",
direction: str = "forward",
limit: Optional[int] = None,
):
# TODO: limit
yield from self.walk(src, dst, edges, "dfs", direction, limit)
@check_arguments
def count_leaves(
self, src: str, edges: str = "*", direction: str = "forward"
) -> int:
return len(list(self.leaves(src, edges, direction)))
@check_arguments
def count_neighbors(
self, src: str, edges: str = "*", direction: str = "forward"
) -> int:
return len(self.graph.get_filtered_neighbors(src, edges, direction))
@check_arguments
def count_visit_nodes(
self, src: str, edges: str = "*", direction: str = "forward"
) -> int:
return len(self.graph.get_subgraph(src, edges, direction))
class Graph:
def __init__(
self, nodes: List[SWHIDlike], edges: List[Tuple[SWHIDlike, SWHIDlike]]
):
self.nodes = [str(node) for node in nodes]
self.forward_edges: Dict[str, List[str]] = {}
self.backward_edges: Dict[str, List[str]] = {}
for node in nodes:
self.forward_edges[str(node)] = []
self.backward_edges[str(node)] = []
for (src, dst) in edges:
self.forward_edges[str(src)].append(str(dst))
self.backward_edges[str(dst)].append(str(src))
def get_filtered_neighbors(
self,
src: str,
edges_fmt: str,
direction: str,
) -> Set[str]:
if direction == "forward":
edges = self.forward_edges
elif direction == "backward":
edges = self.backward_edges
else:
raise GraphArgumentException(f"invalid direction: {direction}")
neighbors = edges.get(src, [])
if edges_fmt == "*":
return set(neighbors)
else:
filtered_neighbors: Set[str] = set()
for edges_fmt_item in edges_fmt.split(","):
(src_fmt, dst_fmt) = edges_fmt_item.split(":")
if src_fmt != "*" and not src.startswith(f"swh:1:{src_fmt}:"):
continue
if dst_fmt == "*":
filtered_neighbors.update(neighbors)
else:
prefix = f"swh:1:{dst_fmt}:"
filtered_neighbors.update(
n for n in neighbors if n.startswith(prefix)
)
return filtered_neighbors
def get_subgraph(self, src: str, edges_fmt: str, direction: str) -> Set[str]:
seen = set()
to_visit = {src}
while to_visit:
node = to_visit.pop()
seen.add(node)
neighbors = set(self.get_filtered_neighbors(node, edges_fmt, direction))
new_nodes = neighbors - seen
to_visit.update(new_nodes)
return seen
def iter_paths_dfs(
self, direction: str, edges_fmt: str, src: str
) -> Iterator[Tuple[str, ...]]:
for (path, node) in DfsSubgraphIterator(self, direction, edges_fmt, src):
yield path + (node,)
def iter_edges_dfs(
self, direction: str, edges_fmt: str, src: str
) -> Iterator[Tuple[str, str]]:
for (path, node) in DfsSubgraphIterator(self, direction, edges_fmt, src):
if len(path) > 0:
yield (path[-1], node)
class SubgraphIterator(Iterator[Tuple[Tuple[str, ...], str]]):
def __init__(self, graph: Graph, direction: str, edges_fmt: str, src: str):
self.graph = graph
self.direction = direction
self.edges_fmt = edges_fmt
self.seen: Set[str] = set()
self.src = src
def more_work(self) -> bool:
raise NotImplementedError()
def pop(self) -> Tuple[Tuple[str, ...], str]:
raise NotImplementedError()
def push(self, new_path: Tuple[str, ...], neighbor: str) -> None:
raise NotImplementedError()
def __next__(self) -> Tuple[Tuple[str, ...], str]:
# Stores (path, next_node)
if not self.more_work():
raise StopIteration()
(path, node) = self.pop()
new_path = path + (node,)
if node not in self.seen:
neighbors = self.graph.get_filtered_neighbors(
node, self.edges_fmt, self.direction
)
# We want to visit the first neighbor first, and to_visit is a stack;
# so we need to reversed() the list of neighbors to get it on top
# of the stack.
for neighbor in reversed(list(neighbors)):
self.push(new_path, neighbor)
self.seen.add(node)
return (path, node)
class DfsSubgraphIterator(SubgraphIterator):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.to_visit: List[Tuple[Tuple[str, ...], str]] = [((), self.src)]
def more_work(self) -> bool:
return bool(self.to_visit)
def pop(self) -> Tuple[Tuple[str, ...], str]:
return self.to_visit.pop()
def push(self, new_path: Tuple[str, ...], neighbor: str) -> None:
self.to_visit.append((new_path, neighbor))
diff --git a/swh/graph/rpc/swhgraph_pb2.py b/swh/graph/rpc/swhgraph_pb2.py
index 04e45ce..b48ad97 100644
--- a/swh/graph/rpc/swhgraph_pb2.py
+++ b/swh/graph/rpc/swhgraph_pb2.py
@@ -1,196 +1,196 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: swh/graph/rpc/swhgraph.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import enum_type_wrapper
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from google.protobuf import field_mask_pb2 as google_dot_protobuf_dot_field__mask__pb2
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cswh/graph/rpc/swhgraph.proto\x12\tswh.graph\x1a google/protobuf/field_mask.proto\"W\n\x0eGetNodeRequest\x12\r\n\x05swhid\x18\x01 \x01(\t\x12-\n\x04mask\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x00\x88\x01\x01\x42\x07\n\x05_mask\"\xd8\x02\n\x10TraversalRequest\x12\x0b\n\x03src\x18\x01 \x03(\t\x12,\n\tdirection\x18\x02 \x01(\x0e\x32\x19.swh.graph.GraphDirection\x12\x12\n\x05\x65\x64ges\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tmax_edges\x18\x04 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\tmin_depth\x18\x05 \x01(\x03H\x02\x88\x01\x01\x12\x16\n\tmax_depth\x18\x06 \x01(\x03H\x03\x88\x01\x01\x12\x30\n\x0creturn_nodes\x18\x07 \x01(\x0b\x32\x15.swh.graph.NodeFilterH\x04\x88\x01\x01\x12-\n\x04mask\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x05\x88\x01\x01\x42\x08\n\x06_edgesB\x0c\n\n_max_edgesB\x0c\n\n_min_depthB\x0c\n\n_max_depthB\x0f\n\r_return_nodesB\x07\n\x05_mask\"\x97\x02\n\x11\x46indPathToRequest\x12\x0b\n\x03src\x18\x01 \x03(\t\x12%\n\x06target\x18\x02 \x01(\x0b\x32\x15.swh.graph.NodeFilter\x12,\n\tdirection\x18\x03 \x01(\x0e\x32\x19.swh.graph.GraphDirection\x12\x12\n\x05\x65\x64ges\x18\x04 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tmax_edges\x18\x05 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\tmax_depth\x18\x06 \x01(\x03H\x02\x88\x01\x01\x12-\n\x04mask\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x03\x88\x01\x01\x42\x08\n\x06_edgesB\x0c\n\n_max_edgesB\x0c\n\n_max_depthB\x07\n\x05_mask\"\x81\x03\n\x16\x46indPathBetweenRequest\x12\x0b\n\x03src\x18\x01 \x03(\t\x12\x0b\n\x03\x64st\x18\x02 \x03(\t\x12,\n\tdirection\x18\x03 \x01(\x0e\x32\x19.swh.graph.GraphDirection\x12\x39\n\x11\x64irection_reverse\x18\x04 \x01(\x0e\x32\x19.swh.graph.GraphDirectionH\x00\x88\x01\x01\x12\x12\n\x05\x65\x64ges\x18\x05 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\redges_reverse\x18\x06 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tmax_edges\x18\x07 \x01(\x03H\x03\x88\x01\x01\x12\x16\n\tmax_depth\x18\x08 \x01(\x03H\x04\x88\x01\x01\x12-\n\x04mask\x18\t \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x05\x88\x01\x01\x42\x14\n\x12_direction_reverseB\x08\n\x06_edgesB\x10\n\x0e_edges_reverseB\x0c\n\n_max_edgesB\x0c\n\n_max_depthB\x07\n\x05_mask\"\xb2\x01\n\nNodeFilter\x12\x12\n\x05types\x18\x01 \x01(\tH\x00\x88\x01\x01\x12%\n\x18min_traversal_successors\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12%\n\x18max_traversal_successors\x18\x03 \x01(\x03H\x02\x88\x01\x01\x42\x08\n\x06_typesB\x1b\n\x19_min_traversal_successorsB\x1b\n\x19_max_traversal_successors\"\x92\x02\n\x04Node\x12\r\n\x05swhid\x18\x01 \x01(\t\x12\'\n\tsuccessor\x18\x02 \x03(\x0b\x32\x14.swh.graph.Successor\x12\x1b\n\x0enum_successors\x18\t \x01(\x03H\x01\x88\x01\x01\x12%\n\x03\x63nt\x18\x03 \x01(\x0b\x32\x16.swh.graph.ContentDataH\x00\x12&\n\x03rev\x18\x05 \x01(\x0b\x32\x17.swh.graph.RevisionDataH\x00\x12%\n\x03rel\x18\x06 \x01(\x0b\x32\x16.swh.graph.ReleaseDataH\x00\x12$\n\x03ori\x18\x08 \x01(\x0b\x32\x15.swh.graph.OriginDataH\x00\x42\x06\n\x04\x64\x61taB\x11\n\x0f_num_successors\"[\n\x04Path\x12\x1d\n\x04node\x18\x01 \x03(\x0b\x32\x0f.swh.graph.Node\x12\x1e\n\x11middle_node_index\x18\x02 \x01(\x05H\x00\x88\x01\x01\x42\x14\n\x12_middle_node_index\"N\n\tSuccessor\x12\x12\n\x05swhid\x18\x01 \x01(\tH\x00\x88\x01\x01\x12#\n\x05label\x18\x02 \x03(\x0b\x32\x14.swh.graph.EdgeLabelB\x08\n\x06_swhid\"U\n\x0b\x43ontentData\x12\x13\n\x06length\x18\x01 \x01(\x03H\x00\x88\x01\x01\x12\x17\n\nis_skipped\x18\x02 \x01(\x08H\x01\x88\x01\x01\x42\t\n\x07_lengthB\r\n\x0b_is_skipped\"\xc6\x02\n\x0cRevisionData\x12\x13\n\x06\x61uthor\x18\x01 \x01(\x03H\x00\x88\x01\x01\x12\x18\n\x0b\x61uthor_date\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1f\n\x12\x61uthor_date_offset\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x16\n\tcommitter\x18\x04 \x01(\x03H\x03\x88\x01\x01\x12\x1b\n\x0e\x63ommitter_date\x18\x05 \x01(\x03H\x04\x88\x01\x01\x12\"\n\x15\x63ommitter_date_offset\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x14\n\x07message\x18\x07 \x01(\x0cH\x06\x88\x01\x01\x42\t\n\x07_authorB\x0e\n\x0c_author_dateB\x15\n\x13_author_date_offsetB\x0c\n\n_committerB\x11\n\x0f_committer_dateB\x18\n\x16_committer_date_offsetB\n\n\x08_message\"\xcd\x01\n\x0bReleaseData\x12\x13\n\x06\x61uthor\x18\x01 \x01(\x03H\x00\x88\x01\x01\x12\x18\n\x0b\x61uthor_date\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1f\n\x12\x61uthor_date_offset\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x11\n\x04name\x18\x04 \x01(\x0cH\x03\x88\x01\x01\x12\x14\n\x07message\x18\x05 \x01(\x0cH\x04\x88\x01\x01\x42\t\n\x07_authorB\x0e\n\x0c_author_dateB\x15\n\x13_author_date_offsetB\x07\n\x05_nameB\n\n\x08_message\"&\n\nOriginData\x12\x10\n\x03url\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x06\n\x04_url\"-\n\tEdgeLabel\x12\x0c\n\x04name\x18\x01 \x01(\x0c\x12\x12\n\npermission\x18\x02 \x01(\x05\"\x1e\n\rCountResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\"\x0e\n\x0cStatsRequest\"\x95\x02\n\rStatsResponse\x12\x11\n\tnum_nodes\x18\x01 \x01(\x03\x12\x11\n\tnum_edges\x18\x02 \x01(\x03\x12\x13\n\x0b\x63ompression\x18\x03 \x01(\x01\x12\x15\n\rbits_per_node\x18\x04 \x01(\x01\x12\x15\n\rbits_per_edge\x18\x05 \x01(\x01\x12\x14\n\x0c\x61vg_locality\x18\x06 \x01(\x01\x12\x14\n\x0cindegree_min\x18\x07 \x01(\x03\x12\x14\n\x0cindegree_max\x18\x08 \x01(\x03\x12\x14\n\x0cindegree_avg\x18\t \x01(\x01\x12\x15\n\routdegree_min\x18\n \x01(\x03\x12\x15\n\routdegree_max\x18\x0b \x01(\x03\x12\x15\n\routdegree_avg\x18\x0c \x01(\x01*+\n\x0eGraphDirection\x12\x0b\n\x07\x46ORWARD\x10\x00\x12\x0c\n\x08\x42\x41\x43KWARD\x10\x01\x32\xcf\x03\n\x10TraversalService\x12\x35\n\x07GetNode\x12\x19.swh.graph.GetNodeRequest\x1a\x0f.swh.graph.Node\x12:\n\x08Traverse\x12\x1b.swh.graph.TraversalRequest\x1a\x0f.swh.graph.Node0\x01\x12;\n\nFindPathTo\x12\x1c.swh.graph.FindPathToRequest\x1a\x0f.swh.graph.Path\x12\x45\n\x0f\x46indPathBetween\x12!.swh.graph.FindPathBetweenRequest\x1a\x0f.swh.graph.Path\x12\x43\n\nCountNodes\x12\x1b.swh.graph.TraversalRequest\x1a\x18.swh.graph.CountResponse\x12\x43\n\nCountEdges\x12\x1b.swh.graph.TraversalRequest\x1a\x18.swh.graph.CountResponse\x12:\n\x05Stats\x12\x17.swh.graph.StatsRequest\x1a\x18.swh.graph.StatsResponseB0\n\x1eorg.softwareheritage.graph.rpcB\x0cGraphServiceP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cswh/graph/rpc/swhgraph.proto\x12\tswh.graph\x1a google/protobuf/field_mask.proto\"W\n\x0eGetNodeRequest\x12\r\n\x05swhid\x18\x01 \x01(\t\x12-\n\x04mask\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x00\x88\x01\x01\x42\x07\n\x05_mask\"\xd8\x02\n\x10TraversalRequest\x12\x0b\n\x03src\x18\x01 \x03(\t\x12,\n\tdirection\x18\x02 \x01(\x0e\x32\x19.swh.graph.GraphDirection\x12\x12\n\x05\x65\x64ges\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tmax_edges\x18\x04 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\tmin_depth\x18\x05 \x01(\x03H\x02\x88\x01\x01\x12\x16\n\tmax_depth\x18\x06 \x01(\x03H\x03\x88\x01\x01\x12\x30\n\x0creturn_nodes\x18\x07 \x01(\x0b\x32\x15.swh.graph.NodeFilterH\x04\x88\x01\x01\x12-\n\x04mask\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x05\x88\x01\x01\x42\x08\n\x06_edgesB\x0c\n\n_max_edgesB\x0c\n\n_min_depthB\x0c\n\n_max_depthB\x0f\n\r_return_nodesB\x07\n\x05_mask\"\x97\x02\n\x11\x46indPathToRequest\x12\x0b\n\x03src\x18\x01 \x03(\t\x12%\n\x06target\x18\x02 \x01(\x0b\x32\x15.swh.graph.NodeFilter\x12,\n\tdirection\x18\x03 \x01(\x0e\x32\x19.swh.graph.GraphDirection\x12\x12\n\x05\x65\x64ges\x18\x04 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tmax_edges\x18\x05 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\tmax_depth\x18\x06 \x01(\x03H\x02\x88\x01\x01\x12-\n\x04mask\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x03\x88\x01\x01\x42\x08\n\x06_edgesB\x0c\n\n_max_edgesB\x0c\n\n_max_depthB\x07\n\x05_mask\"\x81\x03\n\x16\x46indPathBetweenRequest\x12\x0b\n\x03src\x18\x01 \x03(\t\x12\x0b\n\x03\x64st\x18\x02 \x03(\t\x12,\n\tdirection\x18\x03 \x01(\x0e\x32\x19.swh.graph.GraphDirection\x12\x39\n\x11\x64irection_reverse\x18\x04 \x01(\x0e\x32\x19.swh.graph.GraphDirectionH\x00\x88\x01\x01\x12\x12\n\x05\x65\x64ges\x18\x05 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\redges_reverse\x18\x06 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tmax_edges\x18\x07 \x01(\x03H\x03\x88\x01\x01\x12\x16\n\tmax_depth\x18\x08 \x01(\x03H\x04\x88\x01\x01\x12-\n\x04mask\x18\t \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x05\x88\x01\x01\x42\x14\n\x12_direction_reverseB\x08\n\x06_edgesB\x10\n\x0e_edges_reverseB\x0c\n\n_max_edgesB\x0c\n\n_max_depthB\x07\n\x05_mask\"\xb2\x01\n\nNodeFilter\x12\x12\n\x05types\x18\x01 \x01(\tH\x00\x88\x01\x01\x12%\n\x18min_traversal_successors\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12%\n\x18max_traversal_successors\x18\x03 \x01(\x03H\x02\x88\x01\x01\x42\x08\n\x06_typesB\x1b\n\x19_min_traversal_successorsB\x1b\n\x19_max_traversal_successors\"\x92\x02\n\x04Node\x12\r\n\x05swhid\x18\x01 \x01(\t\x12\'\n\tsuccessor\x18\x02 \x03(\x0b\x32\x14.swh.graph.Successor\x12\x1b\n\x0enum_successors\x18\t \x01(\x03H\x01\x88\x01\x01\x12%\n\x03\x63nt\x18\x03 \x01(\x0b\x32\x16.swh.graph.ContentDataH\x00\x12&\n\x03rev\x18\x05 \x01(\x0b\x32\x17.swh.graph.RevisionDataH\x00\x12%\n\x03rel\x18\x06 \x01(\x0b\x32\x16.swh.graph.ReleaseDataH\x00\x12$\n\x03ori\x18\x08 \x01(\x0b\x32\x15.swh.graph.OriginDataH\x00\x42\x06\n\x04\x64\x61taB\x11\n\x0f_num_successors\"U\n\x04Path\x12\x1d\n\x04node\x18\x01 \x03(\x0b\x32\x0f.swh.graph.Node\x12\x1b\n\x0emidpoint_index\x18\x02 \x01(\x05H\x00\x88\x01\x01\x42\x11\n\x0f_midpoint_index\"N\n\tSuccessor\x12\x12\n\x05swhid\x18\x01 \x01(\tH\x00\x88\x01\x01\x12#\n\x05label\x18\x02 \x03(\x0b\x32\x14.swh.graph.EdgeLabelB\x08\n\x06_swhid\"U\n\x0b\x43ontentData\x12\x13\n\x06length\x18\x01 \x01(\x03H\x00\x88\x01\x01\x12\x17\n\nis_skipped\x18\x02 \x01(\x08H\x01\x88\x01\x01\x42\t\n\x07_lengthB\r\n\x0b_is_skipped\"\xc6\x02\n\x0cRevisionData\x12\x13\n\x06\x61uthor\x18\x01 \x01(\x03H\x00\x88\x01\x01\x12\x18\n\x0b\x61uthor_date\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1f\n\x12\x61uthor_date_offset\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x16\n\tcommitter\x18\x04 \x01(\x03H\x03\x88\x01\x01\x12\x1b\n\x0e\x63ommitter_date\x18\x05 \x01(\x03H\x04\x88\x01\x01\x12\"\n\x15\x63ommitter_date_offset\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x14\n\x07message\x18\x07 \x01(\x0cH\x06\x88\x01\x01\x42\t\n\x07_authorB\x0e\n\x0c_author_dateB\x15\n\x13_author_date_offsetB\x0c\n\n_committerB\x11\n\x0f_committer_dateB\x18\n\x16_committer_date_offsetB\n\n\x08_message\"\xcd\x01\n\x0bReleaseData\x12\x13\n\x06\x61uthor\x18\x01 \x01(\x03H\x00\x88\x01\x01\x12\x18\n\x0b\x61uthor_date\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1f\n\x12\x61uthor_date_offset\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x11\n\x04name\x18\x04 \x01(\x0cH\x03\x88\x01\x01\x12\x14\n\x07message\x18\x05 \x01(\x0cH\x04\x88\x01\x01\x42\t\n\x07_authorB\x0e\n\x0c_author_dateB\x15\n\x13_author_date_offsetB\x07\n\x05_nameB\n\n\x08_message\"&\n\nOriginData\x12\x10\n\x03url\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x06\n\x04_url\"-\n\tEdgeLabel\x12\x0c\n\x04name\x18\x01 \x01(\x0c\x12\x12\n\npermission\x18\x02 \x01(\x05\"\x1e\n\rCountResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\"\x0e\n\x0cStatsRequest\"\x9b\x02\n\rStatsResponse\x12\x11\n\tnum_nodes\x18\x01 \x01(\x03\x12\x11\n\tnum_edges\x18\x02 \x01(\x03\x12\x19\n\x11\x63ompression_ratio\x18\x03 \x01(\x01\x12\x15\n\rbits_per_node\x18\x04 \x01(\x01\x12\x15\n\rbits_per_edge\x18\x05 \x01(\x01\x12\x14\n\x0c\x61vg_locality\x18\x06 \x01(\x01\x12\x14\n\x0cindegree_min\x18\x07 \x01(\x03\x12\x14\n\x0cindegree_max\x18\x08 \x01(\x03\x12\x14\n\x0cindegree_avg\x18\t \x01(\x01\x12\x15\n\routdegree_min\x18\n \x01(\x03\x12\x15\n\routdegree_max\x18\x0b \x01(\x03\x12\x15\n\routdegree_avg\x18\x0c \x01(\x01*+\n\x0eGraphDirection\x12\x0b\n\x07\x46ORWARD\x10\x00\x12\x0c\n\x08\x42\x41\x43KWARD\x10\x01\x32\xcf\x03\n\x10TraversalService\x12\x35\n\x07GetNode\x12\x19.swh.graph.GetNodeRequest\x1a\x0f.swh.graph.Node\x12:\n\x08Traverse\x12\x1b.swh.graph.TraversalRequest\x1a\x0f.swh.graph.Node0\x01\x12;\n\nFindPathTo\x12\x1c.swh.graph.FindPathToRequest\x1a\x0f.swh.graph.Path\x12\x45\n\x0f\x46indPathBetween\x12!.swh.graph.FindPathBetweenRequest\x1a\x0f.swh.graph.Path\x12\x43\n\nCountNodes\x12\x1b.swh.graph.TraversalRequest\x1a\x18.swh.graph.CountResponse\x12\x43\n\nCountEdges\x12\x1b.swh.graph.TraversalRequest\x1a\x18.swh.graph.CountResponse\x12:\n\x05Stats\x12\x17.swh.graph.StatsRequest\x1a\x18.swh.graph.StatsResponseB0\n\x1eorg.softwareheritage.graph.rpcB\x0cGraphServiceP\x01\x62\x06proto3')
_GRAPHDIRECTION = DESCRIPTOR.enum_types_by_name['GraphDirection']
GraphDirection = enum_type_wrapper.EnumTypeWrapper(_GRAPHDIRECTION)
FORWARD = 0
BACKWARD = 1
_GETNODEREQUEST = DESCRIPTOR.message_types_by_name['GetNodeRequest']
_TRAVERSALREQUEST = DESCRIPTOR.message_types_by_name['TraversalRequest']
_FINDPATHTOREQUEST = DESCRIPTOR.message_types_by_name['FindPathToRequest']
_FINDPATHBETWEENREQUEST = DESCRIPTOR.message_types_by_name['FindPathBetweenRequest']
_NODEFILTER = DESCRIPTOR.message_types_by_name['NodeFilter']
_NODE = DESCRIPTOR.message_types_by_name['Node']
_PATH = DESCRIPTOR.message_types_by_name['Path']
_SUCCESSOR = DESCRIPTOR.message_types_by_name['Successor']
_CONTENTDATA = DESCRIPTOR.message_types_by_name['ContentData']
_REVISIONDATA = DESCRIPTOR.message_types_by_name['RevisionData']
_RELEASEDATA = DESCRIPTOR.message_types_by_name['ReleaseData']
_ORIGINDATA = DESCRIPTOR.message_types_by_name['OriginData']
_EDGELABEL = DESCRIPTOR.message_types_by_name['EdgeLabel']
_COUNTRESPONSE = DESCRIPTOR.message_types_by_name['CountResponse']
_STATSREQUEST = DESCRIPTOR.message_types_by_name['StatsRequest']
_STATSRESPONSE = DESCRIPTOR.message_types_by_name['StatsResponse']
GetNodeRequest = _reflection.GeneratedProtocolMessageType('GetNodeRequest', (_message.Message,), {
'DESCRIPTOR' : _GETNODEREQUEST,
'__module__' : 'swh.graph.rpc.swhgraph_pb2'
# @@protoc_insertion_point(class_scope:swh.graph.GetNodeRequest)
})
_sym_db.RegisterMessage(GetNodeRequest)
TraversalRequest = _reflection.GeneratedProtocolMessageType('TraversalRequest', (_message.Message,), {
'DESCRIPTOR' : _TRAVERSALREQUEST,
'__module__' : 'swh.graph.rpc.swhgraph_pb2'
# @@protoc_insertion_point(class_scope:swh.graph.TraversalRequest)
})
_sym_db.RegisterMessage(TraversalRequest)
FindPathToRequest = _reflection.GeneratedProtocolMessageType('FindPathToRequest', (_message.Message,), {
'DESCRIPTOR' : _FINDPATHTOREQUEST,
'__module__' : 'swh.graph.rpc.swhgraph_pb2'
# @@protoc_insertion_point(class_scope:swh.graph.FindPathToRequest)
})
_sym_db.RegisterMessage(FindPathToRequest)
FindPathBetweenRequest = _reflection.GeneratedProtocolMessageType('FindPathBetweenRequest', (_message.Message,), {
'DESCRIPTOR' : _FINDPATHBETWEENREQUEST,
'__module__' : 'swh.graph.rpc.swhgraph_pb2'
# @@protoc_insertion_point(class_scope:swh.graph.FindPathBetweenRequest)
})
_sym_db.RegisterMessage(FindPathBetweenRequest)
NodeFilter = _reflection.GeneratedProtocolMessageType('NodeFilter', (_message.Message,), {
'DESCRIPTOR' : _NODEFILTER,
'__module__' : 'swh.graph.rpc.swhgraph_pb2'
# @@protoc_insertion_point(class_scope:swh.graph.NodeFilter)
})
_sym_db.RegisterMessage(NodeFilter)
Node = _reflection.GeneratedProtocolMessageType('Node', (_message.Message,), {
'DESCRIPTOR' : _NODE,
'__module__' : 'swh.graph.rpc.swhgraph_pb2'
# @@protoc_insertion_point(class_scope:swh.graph.Node)
})
_sym_db.RegisterMessage(Node)
Path = _reflection.GeneratedProtocolMessageType('Path', (_message.Message,), {
'DESCRIPTOR' : _PATH,
'__module__' : 'swh.graph.rpc.swhgraph_pb2'
# @@protoc_insertion_point(class_scope:swh.graph.Path)
})
_sym_db.RegisterMessage(Path)
Successor = _reflection.GeneratedProtocolMessageType('Successor', (_message.Message,), {
'DESCRIPTOR' : _SUCCESSOR,
'__module__' : 'swh.graph.rpc.swhgraph_pb2'
# @@protoc_insertion_point(class_scope:swh.graph.Successor)
})
_sym_db.RegisterMessage(Successor)
ContentData = _reflection.GeneratedProtocolMessageType('ContentData', (_message.Message,), {
'DESCRIPTOR' : _CONTENTDATA,
'__module__' : 'swh.graph.rpc.swhgraph_pb2'
# @@protoc_insertion_point(class_scope:swh.graph.ContentData)
})
_sym_db.RegisterMessage(ContentData)
RevisionData = _reflection.GeneratedProtocolMessageType('RevisionData', (_message.Message,), {
'DESCRIPTOR' : _REVISIONDATA,
'__module__' : 'swh.graph.rpc.swhgraph_pb2'
# @@protoc_insertion_point(class_scope:swh.graph.RevisionData)
})
_sym_db.RegisterMessage(RevisionData)
ReleaseData = _reflection.GeneratedProtocolMessageType('ReleaseData', (_message.Message,), {
'DESCRIPTOR' : _RELEASEDATA,
'__module__' : 'swh.graph.rpc.swhgraph_pb2'
# @@protoc_insertion_point(class_scope:swh.graph.ReleaseData)
})
_sym_db.RegisterMessage(ReleaseData)
OriginData = _reflection.GeneratedProtocolMessageType('OriginData', (_message.Message,), {
'DESCRIPTOR' : _ORIGINDATA,
'__module__' : 'swh.graph.rpc.swhgraph_pb2'
# @@protoc_insertion_point(class_scope:swh.graph.OriginData)
})
_sym_db.RegisterMessage(OriginData)
EdgeLabel = _reflection.GeneratedProtocolMessageType('EdgeLabel', (_message.Message,), {
'DESCRIPTOR' : _EDGELABEL,
'__module__' : 'swh.graph.rpc.swhgraph_pb2'
# @@protoc_insertion_point(class_scope:swh.graph.EdgeLabel)
})
_sym_db.RegisterMessage(EdgeLabel)
CountResponse = _reflection.GeneratedProtocolMessageType('CountResponse', (_message.Message,), {
'DESCRIPTOR' : _COUNTRESPONSE,
'__module__' : 'swh.graph.rpc.swhgraph_pb2'
# @@protoc_insertion_point(class_scope:swh.graph.CountResponse)
})
_sym_db.RegisterMessage(CountResponse)
StatsRequest = _reflection.GeneratedProtocolMessageType('StatsRequest', (_message.Message,), {
'DESCRIPTOR' : _STATSREQUEST,
'__module__' : 'swh.graph.rpc.swhgraph_pb2'
# @@protoc_insertion_point(class_scope:swh.graph.StatsRequest)
})
_sym_db.RegisterMessage(StatsRequest)
StatsResponse = _reflection.GeneratedProtocolMessageType('StatsResponse', (_message.Message,), {
'DESCRIPTOR' : _STATSRESPONSE,
'__module__' : 'swh.graph.rpc.swhgraph_pb2'
# @@protoc_insertion_point(class_scope:swh.graph.StatsResponse)
})
_sym_db.RegisterMessage(StatsResponse)
_TRAVERSALSERVICE = DESCRIPTOR.services_by_name['TraversalService']
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\036org.softwareheritage.graph.rpcB\014GraphServiceP\001'
_GRAPHDIRECTION._serialized_start=2853
_GRAPHDIRECTION._serialized_end=2896
_GETNODEREQUEST._serialized_start=77
_GETNODEREQUEST._serialized_end=164
_TRAVERSALREQUEST._serialized_start=167
_TRAVERSALREQUEST._serialized_end=511
_FINDPATHTOREQUEST._serialized_start=514
_FINDPATHTOREQUEST._serialized_end=793
_FINDPATHBETWEENREQUEST._serialized_start=796
_FINDPATHBETWEENREQUEST._serialized_end=1181
_NODEFILTER._serialized_start=1184
_NODEFILTER._serialized_end=1362
_NODE._serialized_start=1365
_NODE._serialized_end=1639
_PATH._serialized_start=1641
- _PATH._serialized_end=1732
- _SUCCESSOR._serialized_start=1734
- _SUCCESSOR._serialized_end=1812
- _CONTENTDATA._serialized_start=1814
- _CONTENTDATA._serialized_end=1899
- _REVISIONDATA._serialized_start=1902
- _REVISIONDATA._serialized_end=2228
- _RELEASEDATA._serialized_start=2231
- _RELEASEDATA._serialized_end=2436
- _ORIGINDATA._serialized_start=2438
- _ORIGINDATA._serialized_end=2476
- _EDGELABEL._serialized_start=2478
- _EDGELABEL._serialized_end=2523
- _COUNTRESPONSE._serialized_start=2525
- _COUNTRESPONSE._serialized_end=2555
- _STATSREQUEST._serialized_start=2557
- _STATSREQUEST._serialized_end=2571
- _STATSRESPONSE._serialized_start=2574
+ _PATH._serialized_end=1726
+ _SUCCESSOR._serialized_start=1728
+ _SUCCESSOR._serialized_end=1806
+ _CONTENTDATA._serialized_start=1808
+ _CONTENTDATA._serialized_end=1893
+ _REVISIONDATA._serialized_start=1896
+ _REVISIONDATA._serialized_end=2222
+ _RELEASEDATA._serialized_start=2225
+ _RELEASEDATA._serialized_end=2430
+ _ORIGINDATA._serialized_start=2432
+ _ORIGINDATA._serialized_end=2470
+ _EDGELABEL._serialized_start=2472
+ _EDGELABEL._serialized_end=2517
+ _COUNTRESPONSE._serialized_start=2519
+ _COUNTRESPONSE._serialized_end=2549
+ _STATSREQUEST._serialized_start=2551
+ _STATSREQUEST._serialized_end=2565
+ _STATSRESPONSE._serialized_start=2568
_STATSRESPONSE._serialized_end=2851
_TRAVERSALSERVICE._serialized_start=2899
_TRAVERSALSERVICE._serialized_end=3362
# @@protoc_insertion_point(module_scope)
diff --git a/swh/graph/rpc/swhgraph_pb2.pyi b/swh/graph/rpc/swhgraph_pb2.pyi
index d81177c..140f704 100644
--- a/swh/graph/rpc/swhgraph_pb2.pyi
+++ b/swh/graph/rpc/swhgraph_pb2.pyi
@@ -1,639 +1,652 @@
"""
@generated by mypy-protobuf. Do not edit manually!
isort:skip_file
"""
import builtins
import google.protobuf.descriptor
import google.protobuf.field_mask_pb2
import google.protobuf.internal.containers
import google.protobuf.internal.enum_type_wrapper
import google.protobuf.message
import typing
import typing_extensions
DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
class _GraphDirection:
ValueType = typing.NewType('ValueType', builtins.int)
V: typing_extensions.TypeAlias = ValueType
class _GraphDirectionEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_GraphDirection.ValueType], builtins.type):
DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
FORWARD: _GraphDirection.ValueType # 0
BACKWARD: _GraphDirection.ValueType # 1
class GraphDirection(_GraphDirection, metaclass=_GraphDirectionEnumTypeWrapper):
pass
FORWARD: GraphDirection.ValueType # 0
BACKWARD: GraphDirection.ValueType # 1
global___GraphDirection = GraphDirection
class GetNodeRequest(google.protobuf.message.Message):
+ """Describe a node to return"""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
SWHID_FIELD_NUMBER: builtins.int
MASK_FIELD_NUMBER: builtins.int
swhid: typing.Text
+ """SWHID of the node to return"""
+
@property
- def mask(self) -> google.protobuf.field_mask_pb2.FieldMask: ...
+ def mask(self) -> google.protobuf.field_mask_pb2.FieldMask:
+ """FieldMask of which fields are to be returned (e.g., "swhid,cnt.length").
+ By default, all fields are returned.
+ """
+ pass
def __init__(self,
*,
swhid: typing.Text = ...,
mask: typing.Optional[google.protobuf.field_mask_pb2.FieldMask] = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["_mask",b"_mask","mask",b"mask"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["_mask",b"_mask","mask",b"mask","swhid",b"swhid"]) -> None: ...
def WhichOneof(self, oneof_group: typing_extensions.Literal["_mask",b"_mask"]) -> typing.Optional[typing_extensions.Literal["mask"]]: ...
global___GetNodeRequest = GetNodeRequest
class TraversalRequest(google.protobuf.message.Message):
"""TraversalRequest describes how a breadth-first traversal should be
performed, and what should be returned to the client.
"""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
SRC_FIELD_NUMBER: builtins.int
DIRECTION_FIELD_NUMBER: builtins.int
EDGES_FIELD_NUMBER: builtins.int
MAX_EDGES_FIELD_NUMBER: builtins.int
MIN_DEPTH_FIELD_NUMBER: builtins.int
MAX_DEPTH_FIELD_NUMBER: builtins.int
RETURN_NODES_FIELD_NUMBER: builtins.int
MASK_FIELD_NUMBER: builtins.int
@property
def src(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[typing.Text]:
- """Set of source nodes"""
+ """Set of source nodes (SWHIDs)"""
pass
direction: global___GraphDirection.ValueType
"""Direction of the graph to traverse. Defaults to FORWARD."""
edges: typing.Text
"""Edge restriction string (e.g. "rev:dir,dir:cnt").
Defaults to "*" (all).
"""
max_edges: builtins.int
"""Maximum number of edges accessed in the traversal, after which it stops.
Defaults to infinite.
"""
min_depth: builtins.int
"""Do not return nodes with a depth lower than this number.
By default, all depths are returned.
"""
max_depth: builtins.int
"""Maximum depth of the traversal, after which it stops.
Defaults to infinite.
"""
@property
def return_nodes(self) -> global___NodeFilter:
"""Filter which nodes will be sent to the stream. By default, all nodes are
returned.
"""
pass
@property
def mask(self) -> google.protobuf.field_mask_pb2.FieldMask:
"""FieldMask of which fields are to be returned (e.g., "swhid,cnt.length").
By default, all fields are returned.
"""
pass
def __init__(self,
*,
src: typing.Optional[typing.Iterable[typing.Text]] = ...,
direction: global___GraphDirection.ValueType = ...,
edges: typing.Optional[typing.Text] = ...,
max_edges: typing.Optional[builtins.int] = ...,
min_depth: typing.Optional[builtins.int] = ...,
max_depth: typing.Optional[builtins.int] = ...,
return_nodes: typing.Optional[global___NodeFilter] = ...,
mask: typing.Optional[google.protobuf.field_mask_pb2.FieldMask] = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["_edges",b"_edges","_mask",b"_mask","_max_depth",b"_max_depth","_max_edges",b"_max_edges","_min_depth",b"_min_depth","_return_nodes",b"_return_nodes","edges",b"edges","mask",b"mask","max_depth",b"max_depth","max_edges",b"max_edges","min_depth",b"min_depth","return_nodes",b"return_nodes"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["_edges",b"_edges","_mask",b"_mask","_max_depth",b"_max_depth","_max_edges",b"_max_edges","_min_depth",b"_min_depth","_return_nodes",b"_return_nodes","direction",b"direction","edges",b"edges","mask",b"mask","max_depth",b"max_depth","max_edges",b"max_edges","min_depth",b"min_depth","return_nodes",b"return_nodes","src",b"src"]) -> None: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_edges",b"_edges"]) -> typing.Optional[typing_extensions.Literal["edges"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_mask",b"_mask"]) -> typing.Optional[typing_extensions.Literal["mask"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_depth",b"_max_depth"]) -> typing.Optional[typing_extensions.Literal["max_depth"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_edges",b"_max_edges"]) -> typing.Optional[typing_extensions.Literal["max_edges"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_min_depth",b"_min_depth"]) -> typing.Optional[typing_extensions.Literal["min_depth"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_return_nodes",b"_return_nodes"]) -> typing.Optional[typing_extensions.Literal["return_nodes"]]: ...
global___TraversalRequest = TraversalRequest
class FindPathToRequest(google.protobuf.message.Message):
"""FindPathToRequest describes a request to find the shortest path between a
set of nodes and a given target criteria, as well as what should be returned
in the path.
"""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
SRC_FIELD_NUMBER: builtins.int
TARGET_FIELD_NUMBER: builtins.int
DIRECTION_FIELD_NUMBER: builtins.int
EDGES_FIELD_NUMBER: builtins.int
MAX_EDGES_FIELD_NUMBER: builtins.int
MAX_DEPTH_FIELD_NUMBER: builtins.int
MASK_FIELD_NUMBER: builtins.int
@property
def src(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[typing.Text]:
- """Set of source nodes"""
+ """Set of source nodes (SWHIDs)"""
pass
@property
def target(self) -> global___NodeFilter:
"""Target criteria, i.e., what constitutes a valid path destination."""
pass
direction: global___GraphDirection.ValueType
"""Direction of the graph to traverse. Defaults to FORWARD."""
edges: typing.Text
"""Edge restriction string (e.g. "rev:dir,dir:cnt").
Defaults to "*" (all).
"""
max_edges: builtins.int
"""Maximum number of edges accessed in the traversal, after which it stops.
Defaults to infinite.
"""
max_depth: builtins.int
"""Maximum depth of the traversal, after which it stops.
Defaults to infinite.
"""
@property
def mask(self) -> google.protobuf.field_mask_pb2.FieldMask:
"""FieldMask of which fields are to be returned (e.g., "swhid,cnt.length").
By default, all fields are returned.
"""
pass
def __init__(self,
*,
src: typing.Optional[typing.Iterable[typing.Text]] = ...,
target: typing.Optional[global___NodeFilter] = ...,
direction: global___GraphDirection.ValueType = ...,
edges: typing.Optional[typing.Text] = ...,
max_edges: typing.Optional[builtins.int] = ...,
max_depth: typing.Optional[builtins.int] = ...,
mask: typing.Optional[google.protobuf.field_mask_pb2.FieldMask] = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["_edges",b"_edges","_mask",b"_mask","_max_depth",b"_max_depth","_max_edges",b"_max_edges","edges",b"edges","mask",b"mask","max_depth",b"max_depth","max_edges",b"max_edges","target",b"target"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["_edges",b"_edges","_mask",b"_mask","_max_depth",b"_max_depth","_max_edges",b"_max_edges","direction",b"direction","edges",b"edges","mask",b"mask","max_depth",b"max_depth","max_edges",b"max_edges","src",b"src","target",b"target"]) -> None: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_edges",b"_edges"]) -> typing.Optional[typing_extensions.Literal["edges"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_mask",b"_mask"]) -> typing.Optional[typing_extensions.Literal["mask"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_depth",b"_max_depth"]) -> typing.Optional[typing_extensions.Literal["max_depth"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_edges",b"_max_edges"]) -> typing.Optional[typing_extensions.Literal["max_edges"]]: ...
global___FindPathToRequest = FindPathToRequest
class FindPathBetweenRequest(google.protobuf.message.Message):
"""FindPathToRequest describes a request to find the shortest path between a
set of source nodes and a set of destination nodes. It works by performing a
bidirectional breadth-first traversal from both sets at the same time.
"""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
SRC_FIELD_NUMBER: builtins.int
DST_FIELD_NUMBER: builtins.int
DIRECTION_FIELD_NUMBER: builtins.int
DIRECTION_REVERSE_FIELD_NUMBER: builtins.int
EDGES_FIELD_NUMBER: builtins.int
EDGES_REVERSE_FIELD_NUMBER: builtins.int
MAX_EDGES_FIELD_NUMBER: builtins.int
MAX_DEPTH_FIELD_NUMBER: builtins.int
MASK_FIELD_NUMBER: builtins.int
@property
def src(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[typing.Text]:
- """Set of source nodes"""
+ """Set of source nodes (SWHIDs)"""
pass
@property
def dst(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[typing.Text]:
- """Set of destination nodes"""
+ """Set of destination nodes (SWHIDs)"""
pass
direction: global___GraphDirection.ValueType
"""Direction of the graph to traverse from the source set. Defaults to
FORWARD.
"""
direction_reverse: global___GraphDirection.ValueType
"""Direction of the graph to traverse from the destination set. Defaults to
the opposite of `direction`. If direction and direction_reverse are
identical, it will find the first common successor of both sets in the
given direction.
"""
edges: typing.Text
"""Edge restriction string for the traversal from the source set.
(e.g. "rev:dir,dir:cnt"). Defaults to "*" (all).
"""
edges_reverse: typing.Text
"""Edge restriction string for the reverse traversal from the destination
set.
If not specified:
- If `edges` is not specified either, defaults to "*"
- If direction == direction_reverse, defaults to `edges`
- If direction != direction_reverse, defaults to the reverse of `edges`
(e.g. "rev:dir" becomes "dir:rev").
"""
max_edges: builtins.int
"""Maximum number of edges accessed in the traversal, after which it stops.
Defaults to infinite.
"""
max_depth: builtins.int
"""Maximum depth of the traversal, after which it stops.
Defaults to infinite.
"""
@property
def mask(self) -> google.protobuf.field_mask_pb2.FieldMask:
"""FieldMask of which fields are to be returned (e.g., "swhid,cnt.length").
By default, all fields are returned.
"""
pass
def __init__(self,
*,
src: typing.Optional[typing.Iterable[typing.Text]] = ...,
dst: typing.Optional[typing.Iterable[typing.Text]] = ...,
direction: global___GraphDirection.ValueType = ...,
direction_reverse: typing.Optional[global___GraphDirection.ValueType] = ...,
edges: typing.Optional[typing.Text] = ...,
edges_reverse: typing.Optional[typing.Text] = ...,
max_edges: typing.Optional[builtins.int] = ...,
max_depth: typing.Optional[builtins.int] = ...,
mask: typing.Optional[google.protobuf.field_mask_pb2.FieldMask] = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["_direction_reverse",b"_direction_reverse","_edges",b"_edges","_edges_reverse",b"_edges_reverse","_mask",b"_mask","_max_depth",b"_max_depth","_max_edges",b"_max_edges","direction_reverse",b"direction_reverse","edges",b"edges","edges_reverse",b"edges_reverse","mask",b"mask","max_depth",b"max_depth","max_edges",b"max_edges"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["_direction_reverse",b"_direction_reverse","_edges",b"_edges","_edges_reverse",b"_edges_reverse","_mask",b"_mask","_max_depth",b"_max_depth","_max_edges",b"_max_edges","direction",b"direction","direction_reverse",b"direction_reverse","dst",b"dst","edges",b"edges","edges_reverse",b"edges_reverse","mask",b"mask","max_depth",b"max_depth","max_edges",b"max_edges","src",b"src"]) -> None: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_direction_reverse",b"_direction_reverse"]) -> typing.Optional[typing_extensions.Literal["direction_reverse"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_edges",b"_edges"]) -> typing.Optional[typing_extensions.Literal["edges"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_edges_reverse",b"_edges_reverse"]) -> typing.Optional[typing_extensions.Literal["edges_reverse"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_mask",b"_mask"]) -> typing.Optional[typing_extensions.Literal["mask"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_depth",b"_max_depth"]) -> typing.Optional[typing_extensions.Literal["max_depth"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_edges",b"_max_edges"]) -> typing.Optional[typing_extensions.Literal["max_edges"]]: ...
global___FindPathBetweenRequest = FindPathBetweenRequest
class NodeFilter(google.protobuf.message.Message):
"""Represents various criteria that make a given node is "valid". A node is
only valid if all the subcriteria present in this message are fulfilled.
"""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
TYPES_FIELD_NUMBER: builtins.int
MIN_TRAVERSAL_SUCCESSORS_FIELD_NUMBER: builtins.int
MAX_TRAVERSAL_SUCCESSORS_FIELD_NUMBER: builtins.int
types: typing.Text
"""Node restriction string. (e.g. "dir,cnt,rev"). Defaults to "*" (all)."""
min_traversal_successors: builtins.int
"""Minimum number of successors encountered *during the traversal*.
Default: no constraint
"""
max_traversal_successors: builtins.int
"""Maximum number of successors encountered *during the traversal*.
Default: no constraint
"""
def __init__(self,
*,
types: typing.Optional[typing.Text] = ...,
min_traversal_successors: typing.Optional[builtins.int] = ...,
max_traversal_successors: typing.Optional[builtins.int] = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["_max_traversal_successors",b"_max_traversal_successors","_min_traversal_successors",b"_min_traversal_successors","_types",b"_types","max_traversal_successors",b"max_traversal_successors","min_traversal_successors",b"min_traversal_successors","types",b"types"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["_max_traversal_successors",b"_max_traversal_successors","_min_traversal_successors",b"_min_traversal_successors","_types",b"_types","max_traversal_successors",b"max_traversal_successors","min_traversal_successors",b"min_traversal_successors","types",b"types"]) -> None: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_traversal_successors",b"_max_traversal_successors"]) -> typing.Optional[typing_extensions.Literal["max_traversal_successors"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_min_traversal_successors",b"_min_traversal_successors"]) -> typing.Optional[typing_extensions.Literal["min_traversal_successors"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_types",b"_types"]) -> typing.Optional[typing_extensions.Literal["types"]]: ...
global___NodeFilter = NodeFilter
class Node(google.protobuf.message.Message):
"""Represents a node in the graph."""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
SWHID_FIELD_NUMBER: builtins.int
SUCCESSOR_FIELD_NUMBER: builtins.int
NUM_SUCCESSORS_FIELD_NUMBER: builtins.int
CNT_FIELD_NUMBER: builtins.int
REV_FIELD_NUMBER: builtins.int
REL_FIELD_NUMBER: builtins.int
ORI_FIELD_NUMBER: builtins.int
swhid: typing.Text
"""The SWHID of the graph node."""
@property
def successor(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Successor]:
"""List of relevant successors of this node."""
pass
num_successors: builtins.int
"""Number of relevant successors."""
@property
def cnt(self) -> global___ContentData: ...
@property
def rev(self) -> global___RevisionData: ...
@property
def rel(self) -> global___ReleaseData: ...
@property
def ori(self) -> global___OriginData: ...
def __init__(self,
*,
swhid: typing.Text = ...,
successor: typing.Optional[typing.Iterable[global___Successor]] = ...,
num_successors: typing.Optional[builtins.int] = ...,
cnt: typing.Optional[global___ContentData] = ...,
rev: typing.Optional[global___RevisionData] = ...,
rel: typing.Optional[global___ReleaseData] = ...,
ori: typing.Optional[global___OriginData] = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["_num_successors",b"_num_successors","cnt",b"cnt","data",b"data","num_successors",b"num_successors","ori",b"ori","rel",b"rel","rev",b"rev"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["_num_successors",b"_num_successors","cnt",b"cnt","data",b"data","num_successors",b"num_successors","ori",b"ori","rel",b"rel","rev",b"rev","successor",b"successor","swhid",b"swhid"]) -> None: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_num_successors",b"_num_successors"]) -> typing.Optional[typing_extensions.Literal["num_successors"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["data",b"data"]) -> typing.Optional[typing_extensions.Literal["cnt","rev","rel","ori"]]: ...
global___Node = Node
class Path(google.protobuf.message.Message):
"""Represents a path in the graph."""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
NODE_FIELD_NUMBER: builtins.int
- MIDDLE_NODE_INDEX_FIELD_NUMBER: builtins.int
+ MIDPOINT_INDEX_FIELD_NUMBER: builtins.int
@property
def node(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Node]:
"""List of nodes in the path, from source to destination"""
pass
- middle_node_index: builtins.int
- """Index of the "middle node" of the path. For paths obtained in
+ midpoint_index: builtins.int
+ """Index of the "midpoint" of the path. For paths obtained with
bidirectional search queries, this is the node that joined the two
sets together. When looking for a common ancestor between two nodes by
performing a FindPathBetween search with two backward graphs, this will
be the index of the common ancestor in the path.
"""
def __init__(self,
*,
node: typing.Optional[typing.Iterable[global___Node]] = ...,
- middle_node_index: typing.Optional[builtins.int] = ...,
+ midpoint_index: typing.Optional[builtins.int] = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["_middle_node_index",b"_middle_node_index","middle_node_index",b"middle_node_index"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["_middle_node_index",b"_middle_node_index","middle_node_index",b"middle_node_index","node",b"node"]) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["_middle_node_index",b"_middle_node_index"]) -> typing.Optional[typing_extensions.Literal["middle_node_index"]]: ...
+ def HasField(self, field_name: typing_extensions.Literal["_midpoint_index",b"_midpoint_index","midpoint_index",b"midpoint_index"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_midpoint_index",b"_midpoint_index","midpoint_index",b"midpoint_index","node",b"node"]) -> None: ...
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["_midpoint_index",b"_midpoint_index"]) -> typing.Optional[typing_extensions.Literal["midpoint_index"]]: ...
global___Path = Path
class Successor(google.protobuf.message.Message):
"""Represents a successor of a given node."""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
SWHID_FIELD_NUMBER: builtins.int
LABEL_FIELD_NUMBER: builtins.int
swhid: typing.Text
"""The SWHID of the successor"""
@property
def label(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___EdgeLabel]:
"""A list of edge labels for the given edge"""
pass
def __init__(self,
*,
swhid: typing.Optional[typing.Text] = ...,
label: typing.Optional[typing.Iterable[global___EdgeLabel]] = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["_swhid",b"_swhid","swhid",b"swhid"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["_swhid",b"_swhid","label",b"label","swhid",b"swhid"]) -> None: ...
def WhichOneof(self, oneof_group: typing_extensions.Literal["_swhid",b"_swhid"]) -> typing.Optional[typing_extensions.Literal["swhid"]]: ...
global___Successor = Successor
class ContentData(google.protobuf.message.Message):
"""Content node properties"""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
LENGTH_FIELD_NUMBER: builtins.int
IS_SKIPPED_FIELD_NUMBER: builtins.int
length: builtins.int
"""Length of the blob, in bytes"""
is_skipped: builtins.bool
"""Whether the content was skipped during ingestion."""
def __init__(self,
*,
length: typing.Optional[builtins.int] = ...,
is_skipped: typing.Optional[builtins.bool] = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["_is_skipped",b"_is_skipped","_length",b"_length","is_skipped",b"is_skipped","length",b"length"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["_is_skipped",b"_is_skipped","_length",b"_length","is_skipped",b"is_skipped","length",b"length"]) -> None: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_is_skipped",b"_is_skipped"]) -> typing.Optional[typing_extensions.Literal["is_skipped"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_length",b"_length"]) -> typing.Optional[typing_extensions.Literal["length"]]: ...
global___ContentData = ContentData
class RevisionData(google.protobuf.message.Message):
"""Revision node properties"""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
AUTHOR_FIELD_NUMBER: builtins.int
AUTHOR_DATE_FIELD_NUMBER: builtins.int
AUTHOR_DATE_OFFSET_FIELD_NUMBER: builtins.int
COMMITTER_FIELD_NUMBER: builtins.int
COMMITTER_DATE_FIELD_NUMBER: builtins.int
COMMITTER_DATE_OFFSET_FIELD_NUMBER: builtins.int
MESSAGE_FIELD_NUMBER: builtins.int
author: builtins.int
"""Revision author ID (anonymized)"""
author_date: builtins.int
"""UNIX timestamp of the revision date (UTC)"""
author_date_offset: builtins.int
"""Timezone of the revision author date as an offset from UTC"""
committer: builtins.int
"""Revision committer ID (anonymized)"""
committer_date: builtins.int
"""UNIX timestamp of the revision committer date (UTC)"""
committer_date_offset: builtins.int
"""Timezone of the revision committer date as an offset from UTC"""
message: builtins.bytes
"""Revision message"""
def __init__(self,
*,
author: typing.Optional[builtins.int] = ...,
author_date: typing.Optional[builtins.int] = ...,
author_date_offset: typing.Optional[builtins.int] = ...,
committer: typing.Optional[builtins.int] = ...,
committer_date: typing.Optional[builtins.int] = ...,
committer_date_offset: typing.Optional[builtins.int] = ...,
message: typing.Optional[builtins.bytes] = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["_author",b"_author","_author_date",b"_author_date","_author_date_offset",b"_author_date_offset","_committer",b"_committer","_committer_date",b"_committer_date","_committer_date_offset",b"_committer_date_offset","_message",b"_message","author",b"author","author_date",b"author_date","author_date_offset",b"author_date_offset","committer",b"committer","committer_date",b"committer_date","committer_date_offset",b"committer_date_offset","message",b"message"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["_author",b"_author","_author_date",b"_author_date","_author_date_offset",b"_author_date_offset","_committer",b"_committer","_committer_date",b"_committer_date","_committer_date_offset",b"_committer_date_offset","_message",b"_message","author",b"author","author_date",b"author_date","author_date_offset",b"author_date_offset","committer",b"committer","committer_date",b"committer_date","committer_date_offset",b"committer_date_offset","message",b"message"]) -> None: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_author",b"_author"]) -> typing.Optional[typing_extensions.Literal["author"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_author_date",b"_author_date"]) -> typing.Optional[typing_extensions.Literal["author_date"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_author_date_offset",b"_author_date_offset"]) -> typing.Optional[typing_extensions.Literal["author_date_offset"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_committer",b"_committer"]) -> typing.Optional[typing_extensions.Literal["committer"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_committer_date",b"_committer_date"]) -> typing.Optional[typing_extensions.Literal["committer_date"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_committer_date_offset",b"_committer_date_offset"]) -> typing.Optional[typing_extensions.Literal["committer_date_offset"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_message",b"_message"]) -> typing.Optional[typing_extensions.Literal["message"]]: ...
global___RevisionData = RevisionData
class ReleaseData(google.protobuf.message.Message):
"""Release node properties"""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
AUTHOR_FIELD_NUMBER: builtins.int
AUTHOR_DATE_FIELD_NUMBER: builtins.int
AUTHOR_DATE_OFFSET_FIELD_NUMBER: builtins.int
NAME_FIELD_NUMBER: builtins.int
MESSAGE_FIELD_NUMBER: builtins.int
author: builtins.int
+ """Release author ID (anonymized)"""
+
author_date: builtins.int
"""UNIX timestamp of the release date (UTC)"""
author_date_offset: builtins.int
"""Timezone of the release author date as an offset from UTC"""
name: builtins.bytes
"""Release name"""
message: builtins.bytes
"""Release message"""
def __init__(self,
*,
author: typing.Optional[builtins.int] = ...,
author_date: typing.Optional[builtins.int] = ...,
author_date_offset: typing.Optional[builtins.int] = ...,
name: typing.Optional[builtins.bytes] = ...,
message: typing.Optional[builtins.bytes] = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["_author",b"_author","_author_date",b"_author_date","_author_date_offset",b"_author_date_offset","_message",b"_message","_name",b"_name","author",b"author","author_date",b"author_date","author_date_offset",b"author_date_offset","message",b"message","name",b"name"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["_author",b"_author","_author_date",b"_author_date","_author_date_offset",b"_author_date_offset","_message",b"_message","_name",b"_name","author",b"author","author_date",b"author_date","author_date_offset",b"author_date_offset","message",b"message","name",b"name"]) -> None: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_author",b"_author"]) -> typing.Optional[typing_extensions.Literal["author"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_author_date",b"_author_date"]) -> typing.Optional[typing_extensions.Literal["author_date"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_author_date_offset",b"_author_date_offset"]) -> typing.Optional[typing_extensions.Literal["author_date_offset"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_message",b"_message"]) -> typing.Optional[typing_extensions.Literal["message"]]: ...
@typing.overload
def WhichOneof(self, oneof_group: typing_extensions.Literal["_name",b"_name"]) -> typing.Optional[typing_extensions.Literal["name"]]: ...
global___ReleaseData = ReleaseData
class OriginData(google.protobuf.message.Message):
"""Origin node properties"""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
URL_FIELD_NUMBER: builtins.int
url: typing.Text
"""URL of the origin"""
def __init__(self,
*,
url: typing.Optional[typing.Text] = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["_url",b"_url","url",b"url"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["_url",b"_url","url",b"url"]) -> None: ...
def WhichOneof(self, oneof_group: typing_extensions.Literal["_url",b"_url"]) -> typing.Optional[typing_extensions.Literal["url"]]: ...
global___OriginData = OriginData
class EdgeLabel(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
NAME_FIELD_NUMBER: builtins.int
PERMISSION_FIELD_NUMBER: builtins.int
name: builtins.bytes
"""Directory entry name for directories, branch name for snapshots"""
permission: builtins.int
"""Entry permission (only set for directories)."""
def __init__(self,
*,
name: builtins.bytes = ...,
permission: builtins.int = ...,
) -> None: ...
def ClearField(self, field_name: typing_extensions.Literal["name",b"name","permission",b"permission"]) -> None: ...
global___EdgeLabel = EdgeLabel
class CountResponse(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
COUNT_FIELD_NUMBER: builtins.int
count: builtins.int
def __init__(self,
*,
count: builtins.int = ...,
) -> None: ...
def ClearField(self, field_name: typing_extensions.Literal["count",b"count"]) -> None: ...
global___CountResponse = CountResponse
class StatsRequest(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
def __init__(self,
) -> None: ...
global___StatsRequest = StatsRequest
class StatsResponse(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
NUM_NODES_FIELD_NUMBER: builtins.int
NUM_EDGES_FIELD_NUMBER: builtins.int
- COMPRESSION_FIELD_NUMBER: builtins.int
+ COMPRESSION_RATIO_FIELD_NUMBER: builtins.int
BITS_PER_NODE_FIELD_NUMBER: builtins.int
BITS_PER_EDGE_FIELD_NUMBER: builtins.int
AVG_LOCALITY_FIELD_NUMBER: builtins.int
INDEGREE_MIN_FIELD_NUMBER: builtins.int
INDEGREE_MAX_FIELD_NUMBER: builtins.int
INDEGREE_AVG_FIELD_NUMBER: builtins.int
OUTDEGREE_MIN_FIELD_NUMBER: builtins.int
OUTDEGREE_MAX_FIELD_NUMBER: builtins.int
OUTDEGREE_AVG_FIELD_NUMBER: builtins.int
num_nodes: builtins.int
+ """Number of nodes in the graph"""
+
num_edges: builtins.int
- compression: builtins.float
+ """Number of edges in the graph"""
+
+ compression_ratio: builtins.float
bits_per_node: builtins.float
bits_per_edge: builtins.float
avg_locality: builtins.float
indegree_min: builtins.int
indegree_max: builtins.int
indegree_avg: builtins.float
outdegree_min: builtins.int
outdegree_max: builtins.int
outdegree_avg: builtins.float
def __init__(self,
*,
num_nodes: builtins.int = ...,
num_edges: builtins.int = ...,
- compression: builtins.float = ...,
+ compression_ratio: builtins.float = ...,
bits_per_node: builtins.float = ...,
bits_per_edge: builtins.float = ...,
avg_locality: builtins.float = ...,
indegree_min: builtins.int = ...,
indegree_max: builtins.int = ...,
indegree_avg: builtins.float = ...,
outdegree_min: builtins.int = ...,
outdegree_max: builtins.int = ...,
outdegree_avg: builtins.float = ...,
) -> None: ...
- def ClearField(self, field_name: typing_extensions.Literal["avg_locality",b"avg_locality","bits_per_edge",b"bits_per_edge","bits_per_node",b"bits_per_node","compression",b"compression","indegree_avg",b"indegree_avg","indegree_max",b"indegree_max","indegree_min",b"indegree_min","num_edges",b"num_edges","num_nodes",b"num_nodes","outdegree_avg",b"outdegree_avg","outdegree_max",b"outdegree_max","outdegree_min",b"outdegree_min"]) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["avg_locality",b"avg_locality","bits_per_edge",b"bits_per_edge","bits_per_node",b"bits_per_node","compression_ratio",b"compression_ratio","indegree_avg",b"indegree_avg","indegree_max",b"indegree_max","indegree_min",b"indegree_min","num_edges",b"num_edges","num_nodes",b"num_nodes","outdegree_avg",b"outdegree_avg","outdegree_max",b"outdegree_max","outdegree_min",b"outdegree_min"]) -> None: ...
global___StatsResponse = StatsResponse
diff --git a/swh/graph/rpc/swhgraph_pb2_grpc.py b/swh/graph/rpc/swhgraph_pb2_grpc.py
index 2f08d12..1ca7ce0 100644
--- a/swh/graph/rpc/swhgraph_pb2_grpc.py
+++ b/swh/graph/rpc/swhgraph_pb2_grpc.py
@@ -1,276 +1,303 @@
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
"""Client and server classes corresponding to protobuf-defined services."""
import grpc
from swh.graph.rpc import swhgraph_pb2 as swh_dot_graph_dot_rpc_dot_swhgraph__pb2
class TraversalServiceStub(object):
- """Missing associated documentation comment in .proto file."""
+ """Graph traversal service
+ """
def __init__(self, channel):
"""Constructor.
Args:
channel: A grpc.Channel.
"""
self.GetNode = channel.unary_unary(
'/swh.graph.TraversalService/GetNode',
request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.GetNodeRequest.SerializeToString,
response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.FromString,
)
self.Traverse = channel.unary_stream(
'/swh.graph.TraversalService/Traverse',
request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString,
response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.FromString,
)
self.FindPathTo = channel.unary_unary(
'/swh.graph.TraversalService/FindPathTo',
request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.FindPathToRequest.SerializeToString,
response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Path.FromString,
)
self.FindPathBetween = channel.unary_unary(
'/swh.graph.TraversalService/FindPathBetween',
request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.FindPathBetweenRequest.SerializeToString,
response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Path.FromString,
)
self.CountNodes = channel.unary_unary(
'/swh.graph.TraversalService/CountNodes',
request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString,
response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.FromString,
)
self.CountEdges = channel.unary_unary(
'/swh.graph.TraversalService/CountEdges',
request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString,
response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.FromString,
)
self.Stats = channel.unary_unary(
'/swh.graph.TraversalService/Stats',
request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsRequest.SerializeToString,
response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsResponse.FromString,
)
class TraversalServiceServicer(object):
- """Missing associated documentation comment in .proto file."""
+ """Graph traversal service
+ """
def GetNode(self, request, context):
"""GetNode returns a single Node and its properties.
"""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def Traverse(self, request, context):
"""Traverse performs a breadth-first graph traversal from a set of source
- nodes, then streams the nodes it encounters, along with their properties.
+ nodes, then streams the nodes it encounters (if they match a given
+ return filter), along with their properties.
"""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def FindPathTo(self, request, context):
"""FindPathTo searches for the shortest path between a set of source nodes
and a node that matches a specific *criteria*.
+
+ It does so by performing a breadth-first search from the source node,
+ until any node that matches the given criteria is found, then follows
+ back its parents to return the shortest path from the source set to that
+ node.
"""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def FindPathBetween(self, request, context):
"""FindPathBetween searches for the shortest path between a set of source
- nodes and a set of destination nodes.
+ nodes and a set of destination nodes.
+
+ It does so by performing a *bidirectional breadth-first search*, i.e.,
+ two parallel breadth-first searches, one from the source set ("src-BFS")
+ and one from the destination set ("dst-BFS"), until both searches find a
+ common node that joins their visited sets. This node is called the
+ "midpoint node".
+ The path returned is the path src -> ... -> midpoint * -> ... -> dst,
+ which is the shortest path between src and dst.
+
+ The graph direction of both BFS can be configured separately. By
+ default, the dst-BFS will use the graph in the opposite direction than
+ the src-BFS (if direction = FORWARD, by default direction_reverse =
+ BACKWARD, and vice-versa). The default behavior is thus to search for
+ the shortest path between two nodes in a given direction. However, one
+ can also specify FORWARD or BACKWARD for *both* the src-BFS and the
+ dst-BFS. This will search for a common descendant or a common ancestor
+ between the two sets, respectively. These will be the midpoints of the
+ returned path.
"""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def CountNodes(self, request, context):
"""CountNodes does the same as Traverse, but only returns the number of
nodes accessed during the traversal.
"""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def CountEdges(self, request, context):
"""CountEdges does the same as Traverse, but only returns the number of
edges accessed during the traversal.
"""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def Stats(self, request, context):
"""Stats returns various
"""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_TraversalServiceServicer_to_server(servicer, server):
rpc_method_handlers = {
'GetNode': grpc.unary_unary_rpc_method_handler(
servicer.GetNode,
request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.GetNodeRequest.FromString,
response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.SerializeToString,
),
'Traverse': grpc.unary_stream_rpc_method_handler(
servicer.Traverse,
request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.FromString,
response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.SerializeToString,
),
'FindPathTo': grpc.unary_unary_rpc_method_handler(
servicer.FindPathTo,
request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.FindPathToRequest.FromString,
response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Path.SerializeToString,
),
'FindPathBetween': grpc.unary_unary_rpc_method_handler(
servicer.FindPathBetween,
request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.FindPathBetweenRequest.FromString,
response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Path.SerializeToString,
),
'CountNodes': grpc.unary_unary_rpc_method_handler(
servicer.CountNodes,
request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.FromString,
response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.SerializeToString,
),
'CountEdges': grpc.unary_unary_rpc_method_handler(
servicer.CountEdges,
request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.FromString,
response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.SerializeToString,
),
'Stats': grpc.unary_unary_rpc_method_handler(
servicer.Stats,
request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsRequest.FromString,
response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsResponse.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'swh.graph.TraversalService', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))
# This class is part of an EXPERIMENTAL API.
class TraversalService(object):
- """Missing associated documentation comment in .proto file."""
+ """Graph traversal service
+ """
@staticmethod
def GetNode(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/GetNode',
swh_dot_graph_dot_rpc_dot_swhgraph__pb2.GetNodeRequest.SerializeToString,
swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@staticmethod
def Traverse(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_stream(request, target, '/swh.graph.TraversalService/Traverse',
swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString,
swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@staticmethod
def FindPathTo(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/FindPathTo',
swh_dot_graph_dot_rpc_dot_swhgraph__pb2.FindPathToRequest.SerializeToString,
swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Path.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@staticmethod
def FindPathBetween(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/FindPathBetween',
swh_dot_graph_dot_rpc_dot_swhgraph__pb2.FindPathBetweenRequest.SerializeToString,
swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Path.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@staticmethod
def CountNodes(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/CountNodes',
swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString,
swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@staticmethod
def CountEdges(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/CountEdges',
swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString,
swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@staticmethod
def Stats(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/Stats',
swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsRequest.SerializeToString,
swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsResponse.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
diff --git a/swh/graph/tests/test_http_client.py b/swh/graph/tests/test_http_client.py
index daaa37c..93240ef 100644
--- a/swh/graph/tests/test_http_client.py
+++ b/swh/graph/tests/test_http_client.py
@@ -1,373 +1,373 @@
import hashlib
import pytest
from pytest import raises
from swh.core.api import RemoteException
from swh.graph.http_client import GraphArgumentException
TEST_ORIGIN_ID = "swh:1:ori:{}".format(
hashlib.sha1(b"https://example.com/swh/graph").hexdigest()
)
def test_stats(graph_client):
stats = graph_client.stats()
assert stats["num_nodes"] == 21
assert stats["num_edges"] == 23
- assert isinstance(stats["compression"], float)
+ assert isinstance(stats["compression_ratio"], float)
assert isinstance(stats["bits_per_node"], float)
assert isinstance(stats["bits_per_edge"], float)
assert isinstance(stats["avg_locality"], float)
assert stats["indegree_min"] == 0
assert stats["indegree_max"] == 3
assert isinstance(stats["indegree_avg"], float)
assert stats["outdegree_min"] == 0
assert stats["outdegree_max"] == 3
assert isinstance(stats["outdegree_avg"], float)
def test_leaves(graph_client):
actual = list(graph_client.leaves(TEST_ORIGIN_ID))
expected = [
"swh:1:cnt:0000000000000000000000000000000000000001",
"swh:1:cnt:0000000000000000000000000000000000000004",
"swh:1:cnt:0000000000000000000000000000000000000005",
"swh:1:cnt:0000000000000000000000000000000000000007",
]
assert set(actual) == set(expected)
def test_neighbors(graph_client):
actual = list(
graph_client.neighbors(
"swh:1:rev:0000000000000000000000000000000000000009", direction="backward"
)
)
expected = [
"swh:1:snp:0000000000000000000000000000000000000020",
"swh:1:rel:0000000000000000000000000000000000000010",
"swh:1:rev:0000000000000000000000000000000000000013",
]
assert set(actual) == set(expected)
def test_visit_nodes(graph_client):
actual = list(
graph_client.visit_nodes(
"swh:1:rel:0000000000000000000000000000000000000010",
edges="rel:rev,rev:rev",
)
)
expected = [
"swh:1:rel:0000000000000000000000000000000000000010",
"swh:1:rev:0000000000000000000000000000000000000009",
"swh:1:rev:0000000000000000000000000000000000000003",
]
assert set(actual) == set(expected)
def test_visit_nodes_filtered(graph_client):
actual = list(
graph_client.visit_nodes(
"swh:1:rel:0000000000000000000000000000000000000010",
return_types="dir",
)
)
expected = [
"swh:1:dir:0000000000000000000000000000000000000002",
"swh:1:dir:0000000000000000000000000000000000000008",
"swh:1:dir:0000000000000000000000000000000000000006",
]
assert set(actual) == set(expected)
def test_visit_nodes_filtered_star(graph_client):
actual = list(
graph_client.visit_nodes(
"swh:1:rel:0000000000000000000000000000000000000010",
return_types="*",
)
)
expected = [
"swh:1:rel:0000000000000000000000000000000000000010",
"swh:1:rev:0000000000000000000000000000000000000009",
"swh:1:rev:0000000000000000000000000000000000000003",
"swh:1:dir:0000000000000000000000000000000000000002",
"swh:1:cnt:0000000000000000000000000000000000000001",
"swh:1:dir:0000000000000000000000000000000000000008",
"swh:1:cnt:0000000000000000000000000000000000000007",
"swh:1:dir:0000000000000000000000000000000000000006",
"swh:1:cnt:0000000000000000000000000000000000000004",
"swh:1:cnt:0000000000000000000000000000000000000005",
]
assert set(actual) == set(expected)
def test_visit_edges(graph_client):
actual = list(
graph_client.visit_edges(
"swh:1:rel:0000000000000000000000000000000000000010",
edges="rel:rev,rev:rev,rev:dir",
)
)
expected = [
(
"swh:1:rel:0000000000000000000000000000000000000010",
"swh:1:rev:0000000000000000000000000000000000000009",
),
(
"swh:1:rev:0000000000000000000000000000000000000009",
"swh:1:rev:0000000000000000000000000000000000000003",
),
(
"swh:1:rev:0000000000000000000000000000000000000009",
"swh:1:dir:0000000000000000000000000000000000000008",
),
(
"swh:1:rev:0000000000000000000000000000000000000003",
"swh:1:dir:0000000000000000000000000000000000000002",
),
]
assert set(actual) == set(expected)
def test_visit_edges_limited(graph_client):
actual = list(
graph_client.visit_edges(
"swh:1:rel:0000000000000000000000000000000000000010",
max_edges=4,
edges="rel:rev,rev:rev,rev:dir",
)
)
expected = [
(
"swh:1:rel:0000000000000000000000000000000000000010",
"swh:1:rev:0000000000000000000000000000000000000009",
),
(
"swh:1:rev:0000000000000000000000000000000000000009",
"swh:1:rev:0000000000000000000000000000000000000003",
),
(
"swh:1:rev:0000000000000000000000000000000000000009",
"swh:1:dir:0000000000000000000000000000000000000008",
),
(
"swh:1:rev:0000000000000000000000000000000000000003",
"swh:1:dir:0000000000000000000000000000000000000002",
),
]
# As there are four valid answers (up to reordering), we cannot check for
# equality. Instead, we check the client returned all edges but one.
assert set(actual).issubset(set(expected))
assert len(actual) == 3
def test_visit_edges_diamond_pattern(graph_client):
actual = list(
graph_client.visit_edges(
"swh:1:rev:0000000000000000000000000000000000000009",
edges="*",
)
)
expected = [
(
"swh:1:rev:0000000000000000000000000000000000000009",
"swh:1:rev:0000000000000000000000000000000000000003",
),
(
"swh:1:rev:0000000000000000000000000000000000000009",
"swh:1:dir:0000000000000000000000000000000000000008",
),
(
"swh:1:rev:0000000000000000000000000000000000000003",
"swh:1:dir:0000000000000000000000000000000000000002",
),
(
"swh:1:dir:0000000000000000000000000000000000000002",
"swh:1:cnt:0000000000000000000000000000000000000001",
),
(
"swh:1:dir:0000000000000000000000000000000000000008",
"swh:1:cnt:0000000000000000000000000000000000000001",
),
(
"swh:1:dir:0000000000000000000000000000000000000008",
"swh:1:cnt:0000000000000000000000000000000000000007",
),
(
"swh:1:dir:0000000000000000000000000000000000000008",
"swh:1:dir:0000000000000000000000000000000000000006",
),
(
"swh:1:dir:0000000000000000000000000000000000000006",
"swh:1:cnt:0000000000000000000000000000000000000004",
),
(
"swh:1:dir:0000000000000000000000000000000000000006",
"swh:1:cnt:0000000000000000000000000000000000000005",
),
]
assert set(actual) == set(expected)
@pytest.mark.skip(reason="currently disabled due to T1969")
def test_walk(graph_client):
args = ("swh:1:dir:0000000000000000000000000000000000000016", "rel")
kwargs = {
"edges": "dir:dir,dir:rev,rev:*",
"direction": "backward",
"traversal": "bfs",
}
actual = list(graph_client.walk(*args, **kwargs))
expected = [
"swh:1:dir:0000000000000000000000000000000000000016",
"swh:1:dir:0000000000000000000000000000000000000017",
"swh:1:rev:0000000000000000000000000000000000000018",
"swh:1:rel:0000000000000000000000000000000000000019",
]
assert set(actual) == set(expected)
kwargs2 = kwargs.copy()
kwargs2["limit"] = -1
actual = list(graph_client.walk(*args, **kwargs2))
expected = ["swh:1:rel:0000000000000000000000000000000000000019"]
assert set(actual) == set(expected)
kwargs2 = kwargs.copy()
kwargs2["limit"] = 2
actual = list(graph_client.walk(*args, **kwargs2))
expected = [
"swh:1:dir:0000000000000000000000000000000000000016",
"swh:1:dir:0000000000000000000000000000000000000017",
]
assert set(actual) == set(expected)
@pytest.mark.skip(reason="Random walk is deprecated")
def test_random_walk_dst_is_type(graph_client):
"""as the walk is random, we test a visit from a cnt node to a release
reachable from every single path in the backward graph, and only check the
final node of the path (i.e., the release)
"""
args = ("swh:1:cnt:0000000000000000000000000000000000000015", "rel")
kwargs = {"direction": "backward"}
expected_root = "swh:1:rel:0000000000000000000000000000000000000019"
actual = list(graph_client.random_walk(*args, **kwargs))
assert len(actual) > 1 # no release directly links to a content
assert actual[0] == args[0]
assert actual[-1] == expected_root
kwargs2 = kwargs.copy()
kwargs2["limit"] = -1
actual = list(graph_client.random_walk(*args, **kwargs2))
assert actual == [expected_root]
kwargs2["limit"] = -2
actual = list(graph_client.random_walk(*args, **kwargs2))
assert len(actual) == 2
assert actual[-1] == expected_root
kwargs2["limit"] = 3
actual = list(graph_client.random_walk(*args, **kwargs2))
assert len(actual) == 3
@pytest.mark.skip(reason="Random walk is deprecated")
def test_random_walk_dst_is_node(graph_client):
"""Same as test_random_walk_dst_is_type, but we target the specific release
node instead of a type
"""
args = (
"swh:1:cnt:0000000000000000000000000000000000000015",
"swh:1:rel:0000000000000000000000000000000000000019",
)
kwargs = {"direction": "backward"}
expected_root = "swh:1:rel:0000000000000000000000000000000000000019"
actual = list(graph_client.random_walk(*args, **kwargs))
assert len(actual) > 1 # no origin directly links to a content
assert actual[0] == args[0]
assert actual[-1] == expected_root
kwargs2 = kwargs.copy()
kwargs2["limit"] = -1
actual = list(graph_client.random_walk(*args, **kwargs2))
assert actual == [expected_root]
kwargs2["limit"] = -2
actual = list(graph_client.random_walk(*args, **kwargs2))
assert len(actual) == 2
assert actual[-1] == expected_root
kwargs2["limit"] = 3
actual = list(graph_client.random_walk(*args, **kwargs2))
assert len(actual) == 3
def test_count(graph_client):
actual = graph_client.count_leaves(TEST_ORIGIN_ID)
assert actual == 4
actual = graph_client.count_visit_nodes(
"swh:1:rel:0000000000000000000000000000000000000010", edges="rel:rev,rev:rev"
)
assert actual == 3
actual = graph_client.count_neighbors(
"swh:1:rev:0000000000000000000000000000000000000009", direction="backward"
)
assert actual == 3
def test_param_validation(graph_client):
with raises(GraphArgumentException) as exc_info: # SWHID not found
list(graph_client.leaves("swh:1:rel:00ffffffff000000000000000000000000000010"))
if exc_info.value.response:
assert exc_info.value.response.status_code == 404
with raises(GraphArgumentException) as exc_info: # malformed SWHID
list(
graph_client.neighbors("swh:1:rel:00ffffffff00000000zzzzzzz000000000000010")
)
if exc_info.value.response:
assert exc_info.value.response.status_code == 400
with raises(GraphArgumentException) as exc_info: # malformed edge specificaiton
list(
graph_client.visit_nodes(
"swh:1:dir:0000000000000000000000000000000000000016",
edges="dir:notanodetype,dir:rev,rev:*",
direction="backward",
)
)
if exc_info.value.response:
assert exc_info.value.response.status_code == 400
with raises(GraphArgumentException) as exc_info: # malformed direction
list(
graph_client.visit_nodes(
"swh:1:dir:0000000000000000000000000000000000000016",
edges="dir:dir,dir:rev,rev:*",
direction="notadirection",
)
)
if exc_info.value.response:
assert exc_info.value.response.status_code == 400
@pytest.mark.skip(reason="currently disabled due to T1969")
def test_param_validation_walk(graph_client):
"""test validation of walk-specific parameters only"""
with raises(RemoteException) as exc_info: # malformed traversal order
list(
graph_client.walk(
"swh:1:dir:0000000000000000000000000000000000000016",
"rel",
edges="dir:dir,dir:rev,rev:*",
direction="backward",
traversal="notatraversalorder",
)
)
assert exc_info.value.response.status_code == 400
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Thu, Jul 3, 10:35 AM (2 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3272883
Attached To
rDGRPH Compressed graph representation
Event Timeline
Log In to Comment