diff --git a/java/pom.xml b/java/pom.xml index d51e6e2..02e438f 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -1,398 +1,403 @@ 4.0.0 org.softwareheritage.graph swh-graph ${git.closest.tag.name} swh-graph https://forge.softwareheritage.org/source/swh-graph/ UTF-8 11 - 3.20.1 - 1.46.0 + 3.21.1 + 1.47.0 ch.qos.logback logback-classic 1.2.3 org.junit.jupiter junit-jupiter-api 5.7.0 test org.junit.jupiter junit-jupiter-engine 5.7.0 test org.slf4j slf4j-simple 1.7.26 it.unimi.dsi webgraph-big 3.6.7 it.unimi.dsi fastutil 8.5.8 it.unimi.dsi dsiutils 2.7.1 it.unimi.dsi sux4j 5.3.1 it.unimi.dsi law 2.7.2 org.apache.hadoop hadoop-common org.umlgraph umlgraph org.eclipse.jetty.aggregate jetty-all it.unimi.di mg4j it.unimi.di mg4j-big com.martiansoftware jsap 2.1 commons-codec commons-codec 1.15 com.github.luben zstd-jni 1.5.1-1 org.apache.orc orc-core 1.7.1 org.apache.hadoop hadoop-common 3.3.1 org.apache.hadoop hadoop-client-runtime 3.3.1 com.google.protobuf protobuf-java ${protobuf.version} io.grpc grpc-netty-shaded ${grpc.version} io.grpc grpc-protobuf ${grpc.version} io.grpc grpc-stub ${grpc.version} io.grpc grpc-services ${grpc.version} io.grpc grpc-testing ${grpc.version} javax.annotation javax.annotation-api 1.3.2 + + com.google.protobuf + protobuf-java-util + ${protobuf.version} + maven-clean-plugin 3.1.0 maven-resources-plugin 3.0.2 maven-compiler-plugin 3.8.0 11 11 -verbose -Xlint:all maven-surefire-plugin 2.22.2 maven-failsafe-plugin 2.22.2 maven-jar-plugin 3.0.2 maven-install-plugin 2.5.2 maven-deploy-plugin 2.8.2 maven-site-plugin 3.7.1 maven-project-info-reports-plugin 3.0.0 maven-dependency-plugin 3.1.2 maven-assembly-plugin 3.3.0 org.softwareheritage.graph.rpc.GraphServer jar-with-dependencies false make-assembly package single com.diffplug.spotless spotless-maven-plugin 2.22.1 *.md .gitignore true 4 4.16.0 .coding-style.xml pl.project13.maven git-commit-id-plugin 3.0.1 get-the-git-infos revision initialize true true true true v* git.closest.tag.name ^v true maven-source-plugin 2.1.1 bundle-sources package jar-no-fork test-jar-no-fork org.apache.maven.plugins maven-javadoc-plugin 3.3.1 resource-bundles package resource-bundle test-resource-bundle false javadoc-jar package jar true it.unimi.dsi:webgraph-big:* https://webgraph.di.unimi.it/docs-big/ https://dsiutils.di.unimi.it/docs/ https://fastutil.di.unimi.it/docs/ https://law.di.unimi.it/software/law-docs/ implSpec a Implementation Requirements: implNote a Implementation Note: org.xolstice.maven.plugins protobuf-maven-plugin 0.6.1 com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} grpc-java io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} compile compile-custom test-compile test-compile-custom kr.motd.maven os-maven-plugin 1.6.2 diff --git a/java/src/main/java/org/softwareheritage/graph/rpc/GraphServer.java b/java/src/main/java/org/softwareheritage/graph/rpc/GraphServer.java index 57a18e1..a1a2c20 100644 --- a/java/src/main/java/org/softwareheritage/graph/rpc/GraphServer.java +++ b/java/src/main/java/org/softwareheritage/graph/rpc/GraphServer.java @@ -1,203 +1,225 @@ package org.softwareheritage.graph.rpc; +import com.google.protobuf.FieldMask; import com.martiansoftware.jsap.*; import io.grpc.Server; +import io.grpc.Status; import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; import io.grpc.netty.shaded.io.netty.channel.ChannelOption; import io.grpc.stub.StreamObserver; import io.grpc.protobuf.services.ProtoReflectionService; import it.unimi.dsi.logging.ProgressLogger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.softwareheritage.graph.SWHID; import org.softwareheritage.graph.SwhBidirectionalGraph; import org.softwareheritage.graph.compress.LabelMapBuilder; import java.io.FileInputStream; import java.io.IOException; import java.util.Properties; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; /** * Server that manages startup/shutdown of a {@code Greeter} server. */ public class GraphServer { private final static Logger logger = LoggerFactory.getLogger(GraphServer.class); private final SwhBidirectionalGraph graph; private final int port; private final int threads; private Server server; public GraphServer(String graphBasename, int port, int threads) throws IOException { - // TODO: use loadLabelledMapped() when https://github.com/vigna/webgraph-big/pull/5 is merged - this.graph = SwhBidirectionalGraph.loadLabelled(graphBasename, new ProgressLogger(logger)); + this.graph = loadGraph(graphBasename); this.port = port; this.threads = threads; - graph.loadContentLength(); - graph.loadContentIsSkipped(); - graph.loadPersonIds(); - graph.loadAuthorTimestamps(); - graph.loadCommitterTimestamps(); - graph.loadMessages(); - graph.loadTagNames(); - graph.loadLabelNames(); + } + + public static SwhBidirectionalGraph loadGraph(String basename) throws IOException { + // TODO: use loadLabelledMapped() when https://github.com/vigna/webgraph-big/pull/5 is merged + SwhBidirectionalGraph g = SwhBidirectionalGraph.loadLabelled(basename, new ProgressLogger(logger)); + g.loadContentLength(); + g.loadContentIsSkipped(); + g.loadPersonIds(); + g.loadAuthorTimestamps(); + g.loadCommitterTimestamps(); + g.loadMessages(); + g.loadTagNames(); + g.loadLabelNames(); + return g; } private void start() throws IOException { server = NettyServerBuilder.forPort(port).withChildOption(ChannelOption.SO_REUSEADDR, true) .executor(Executors.newFixedThreadPool(threads)).addService(new TraversalService(graph)) .addService(ProtoReflectionService.newInstance()).build().start(); logger.info("Server started, listening on " + port); Runtime.getRuntime().addShutdownHook(new Thread(() -> { try { GraphServer.this.stop(); } catch (InterruptedException e) { e.printStackTrace(System.err); } })); } private void stop() throws InterruptedException { if (server != null) { server.shutdown().awaitTermination(30, TimeUnit.SECONDS); } } /** * Await termination on the main thread since the grpc library uses daemon threads. */ private void blockUntilShutdown() throws InterruptedException { if (server != null) { server.awaitTermination(); } } private static JSAPResult parseArgs(String[] args) { JSAPResult config = null; try { SimpleJSAP jsap = new SimpleJSAP(LabelMapBuilder.class.getName(), "", new Parameter[]{ new FlaggedOption("port", JSAP.INTEGER_PARSER, "50091", JSAP.NOT_REQUIRED, 'p', "port", "The port on which the server should listen."), new FlaggedOption("threads", JSAP.INTEGER_PARSER, "1", JSAP.NOT_REQUIRED, 't', "threads", "The number of concurrent threads. 0 = number of cores."), new UnflaggedOption("graphBasename", JSAP.STRING_PARSER, JSAP.REQUIRED, "Basename of the output graph")}); config = jsap.parse(args); if (jsap.messagePrinted()) { System.exit(1); } } catch (JSAPException e) { e.printStackTrace(); } return config; } /** * Main launches the server from the command line. */ public static void main(String[] args) throws IOException, InterruptedException { JSAPResult config = parseArgs(args); String graphBasename = config.getString("graphBasename"); int port = config.getInt("port"); int threads = config.getInt("threads"); if (threads == 0) { threads = Runtime.getRuntime().availableProcessors(); } final GraphServer server = new GraphServer(graphBasename, port, threads); server.start(); server.blockUntilShutdown(); } static class TraversalService extends TraversalServiceGrpc.TraversalServiceImplBase { SwhBidirectionalGraph graph; public TraversalService(SwhBidirectionalGraph graph) { this.graph = graph; } @Override public void checkSwhid(CheckSwhidRequest request, StreamObserver responseObserver) { - boolean exists = true; CheckSwhidResponse.Builder builder = CheckSwhidResponse.newBuilder().setExists(true); try { graph.getNodeId(new SWHID(request.getSwhid())); } catch (IllegalArgumentException e) { builder.setExists(false); builder.setDetails(e.getMessage()); } responseObserver.onNext(builder.build()); responseObserver.onCompleted(); } @Override public void stats(StatsRequest request, StreamObserver responseObserver) { StatsResponse.Builder response = StatsResponse.newBuilder(); response.setNumNodes(graph.numNodes()); response.setNumEdges(graph.numArcs()); Properties properties = new Properties(); try { properties.load(new FileInputStream(graph.getPath() + ".properties")); properties.load(new FileInputStream(graph.getPath() + ".stats")); } catch (IOException e) { throw new RuntimeException(e); } response.setCompression(Double.parseDouble(properties.getProperty("compratio"))); response.setBitsPerNode(Double.parseDouble(properties.getProperty("bitspernode"))); response.setBitsPerEdge(Double.parseDouble(properties.getProperty("bitsperlink"))); response.setAvgLocality(Double.parseDouble(properties.getProperty("avglocality"))); response.setIndegreeMin(Long.parseLong(properties.getProperty("minindegree"))); response.setIndegreeMax(Long.parseLong(properties.getProperty("maxindegree"))); response.setIndegreeAvg(Double.parseDouble(properties.getProperty("avgindegree"))); response.setOutdegreeMin(Long.parseLong(properties.getProperty("minoutdegree"))); response.setOutdegreeMax(Long.parseLong(properties.getProperty("maxoutdegree"))); response.setOutdegreeAvg(Double.parseDouble(properties.getProperty("avgoutdegree"))); responseObserver.onNext(response.build()); responseObserver.onCompleted(); } + @Override + public void getNode(GetNodeRequest request, StreamObserver responseObserver) { + long nodeId; + try { + nodeId = graph.getNodeId(new SWHID(request.getSwhid())); + + } catch (IllegalArgumentException e) { + responseObserver.onError(Status.INVALID_ARGUMENT.withCause(e).asException()); + return; + } + Node.Builder builder = Node.newBuilder(); + NodePropertyBuilder.buildNodeProperties(graph.getForwardGraph(), + request.hasMask() ? request.getMask() : null, builder, nodeId); + responseObserver.onNext(builder.build()); + responseObserver.onCompleted(); + } + @Override public void traverse(TraversalRequest request, StreamObserver responseObserver) { SwhBidirectionalGraph g = graph.copy(); Traversal.simpleTraversal(g, request, responseObserver::onNext); responseObserver.onCompleted(); } @Override public void countNodes(TraversalRequest request, StreamObserver responseObserver) { AtomicInteger count = new AtomicInteger(0); SwhBidirectionalGraph g = graph.copy(); TraversalRequest fixedReq = TraversalRequest.newBuilder(request) // Ignore return fields, just count nodes - .setReturnFields(NodeFields.getDefaultInstance()).build(); + .setMask(FieldMask.getDefaultInstance()).build(); Traversal.simpleTraversal(g, fixedReq, (Node node) -> { count.incrementAndGet(); }); CountResponse response = CountResponse.newBuilder().setCount(count.get()).build(); responseObserver.onNext(response); responseObserver.onCompleted(); } @Override public void countEdges(TraversalRequest request, StreamObserver responseObserver) { AtomicInteger count = new AtomicInteger(0); SwhBidirectionalGraph g = graph.copy(); TraversalRequest fixedReq = TraversalRequest.newBuilder(request) // Force return empty successors to count the edges - .setReturnFields(NodeFields.newBuilder().setSuccessor(true).setSuccessorSwhid(false).build()) - .build(); + .setMask(FieldMask.newBuilder().addPaths("successor").build()).build(); Traversal.simpleTraversal(g, fixedReq, (Node node) -> { count.addAndGet(node.getSuccessorCount()); }); CountResponse response = CountResponse.newBuilder().setCount(count.get()).build(); responseObserver.onNext(response); responseObserver.onCompleted(); } } } diff --git a/java/src/main/java/org/softwareheritage/graph/rpc/NodePropertyBuilder.java b/java/src/main/java/org/softwareheritage/graph/rpc/NodePropertyBuilder.java new file mode 100644 index 0000000..8a13e2b --- /dev/null +++ b/java/src/main/java/org/softwareheritage/graph/rpc/NodePropertyBuilder.java @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2022 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +package org.softwareheritage.graph.rpc; + +import com.google.protobuf.ByteString; +import com.google.protobuf.FieldMask; +import com.google.protobuf.util.FieldMaskUtil; +import it.unimi.dsi.big.webgraph.labelling.Label; +import org.softwareheritage.graph.SwhUnidirectionalGraph; +import org.softwareheritage.graph.labels.DirEntry; + +import java.util.*; + +public class NodePropertyBuilder { + public static class NodeDataMask { + public boolean swhid; + public boolean successor; + public boolean successorSwhid; + public boolean successorLabel; + public boolean cntLength; + public boolean cntIsSkipped; + public boolean revAuthor; + public boolean revAuthorDate; + public boolean revAuthorDateOffset; + public boolean revCommitter; + public boolean revCommitterDate; + public boolean revCommitterDateOffset; + public boolean revMessage; + public boolean relAuthor; + public boolean relAuthorDate; + public boolean relAuthorDateOffset; + public boolean relName; + public boolean relMessage; + public boolean oriUrl; + + public NodeDataMask(FieldMask mask) { + Set allowedFields = null; + if (mask != null) { + mask = FieldMaskUtil.normalize(mask); + allowedFields = new HashSet<>(mask.getPathsList()); + } + this.swhid = allowedFields == null || allowedFields.contains("swhid"); + this.successorSwhid = allowedFields == null || allowedFields.contains("successor") + || allowedFields.contains("successor.swhid"); + this.successorLabel = allowedFields == null || allowedFields.contains("successor") + || allowedFields.contains("successor.label"); + this.successor = this.successorSwhid || this.successorLabel; + this.cntLength = allowedFields == null || allowedFields.contains("cnt.length"); + this.cntIsSkipped = allowedFields == null || allowedFields.contains("cnt.is_skipped"); + this.revAuthor = allowedFields == null || allowedFields.contains("rev.author"); + this.revAuthorDate = allowedFields == null || allowedFields.contains("rev.author_date"); + this.revAuthorDateOffset = allowedFields == null || allowedFields.contains("rev.author_date_offset"); + this.revCommitter = allowedFields == null || allowedFields.contains("rev.committer"); + this.revCommitterDate = allowedFields == null || allowedFields.contains("rev.committer_date"); + this.revCommitterDateOffset = allowedFields == null || allowedFields.contains("rev.committer_date_offset"); + this.revMessage = allowedFields == null || allowedFields.contains("rev.message"); + this.relAuthor = allowedFields == null || allowedFields.contains("rel.author"); + this.relAuthorDate = allowedFields == null || allowedFields.contains("rel.author_date"); + this.relAuthorDateOffset = allowedFields == null || allowedFields.contains("rel.author_date_offset"); + this.relName = allowedFields == null || allowedFields.contains("rel.name"); + this.relMessage = allowedFields == null || allowedFields.contains("rel.message"); + this.oriUrl = allowedFields == null || allowedFields.contains("ori.url"); + } + } + + public static void buildNodeProperties(SwhUnidirectionalGraph graph, NodeDataMask mask, Node.Builder nodeBuilder, + long node) { + if (mask.swhid) { + nodeBuilder.setSwhid(graph.getSWHID(node).toString()); + } + + switch (graph.getNodeType(node)) { + case CNT: + ContentData.Builder cntBuilder = ContentData.newBuilder(); + if (mask.cntLength) { + cntBuilder.setLength(graph.getContentLength(node)); + } + if (mask.cntIsSkipped) { + cntBuilder.setIsSkipped(graph.isContentSkipped(node)); + } + nodeBuilder.setCnt(cntBuilder.build()); + break; + case REV: + RevisionData.Builder revBuilder = RevisionData.newBuilder(); + if (mask.revAuthor) { + revBuilder.setAuthor(graph.getAuthorId(node)); + } + if (mask.revAuthorDate) { + revBuilder.setAuthorDate(graph.getAuthorTimestamp(node)); + } + if (mask.revAuthorDateOffset) { + revBuilder.setAuthorDateOffset(graph.getAuthorTimestampOffset(node)); + } + if (mask.revCommitter) { + revBuilder.setCommitter(graph.getCommitterId(node)); + } + if (mask.revCommitterDate) { + revBuilder.setCommitterDate(graph.getCommitterTimestamp(node)); + } + if (mask.revCommitterDateOffset) { + revBuilder.setCommitterDateOffset(graph.getCommitterTimestampOffset(node)); + } + if (mask.revMessage) { + byte[] msg = graph.getMessage(node); + if (msg != null) { + revBuilder.setMessage(ByteString.copyFrom(msg)); + } + } + nodeBuilder.setRev(revBuilder.build()); + break; + case REL: + ReleaseData.Builder relBuilder = ReleaseData.newBuilder(); + if (mask.relAuthor) { + relBuilder.setAuthor(graph.getAuthorId(node)); + } + if (mask.relAuthorDate) { + relBuilder.setAuthorDate(graph.getAuthorTimestamp(node)); + } + if (mask.relAuthorDateOffset) { + relBuilder.setAuthorDateOffset(graph.getAuthorTimestampOffset(node)); + } + if (mask.relName) { + byte[] msg = graph.getMessage(node); + if (msg != null) { + relBuilder.setMessage(ByteString.copyFrom(msg)); + } + } + if (mask.relMessage) { + byte[] msg = graph.getMessage(node); + if (msg != null) { + relBuilder.setMessage(ByteString.copyFrom(msg)); + } + } + nodeBuilder.setRel(relBuilder.build()); + break; + case ORI: + OriginData.Builder oriBuilder = OriginData.newBuilder(); + if (mask.oriUrl) { + String url = graph.getUrl(node); + if (url != null) { + oriBuilder.setUrl(url); + } + } + nodeBuilder.setOri(oriBuilder.build()); + } + } + + public static void buildNodeProperties(SwhUnidirectionalGraph graph, FieldMask mask, Node.Builder nodeBuilder, + long node) { + NodeDataMask nodeMask = new NodeDataMask(mask); + buildNodeProperties(graph, nodeMask, nodeBuilder, node); + } + + public static void buildSuccessorProperties(SwhUnidirectionalGraph graph, NodeDataMask mask, + Node.Builder nodeBuilder, long src, long dst, Label label) { + if (nodeBuilder != null && mask.successor) { + Successor.Builder successorBuilder = Successor.newBuilder(); + if (mask.successorSwhid) { + successorBuilder.setSwhid(graph.getSWHID(dst).toString()); + } + if (mask.successorLabel) { + DirEntry[] entries = (DirEntry[]) label.get(); + for (DirEntry entry : entries) { + EdgeLabel.Builder builder = EdgeLabel.newBuilder(); + builder.setName(ByteString.copyFrom(graph.getLabelName(entry.filenameId))); + builder.setPermission(entry.permission); + successorBuilder.addLabel(builder.build()); + } + } + nodeBuilder.addSuccessor(successorBuilder.build()); + } + } + + public static void buildSuccessorProperties(SwhUnidirectionalGraph graph, FieldMask mask, Node.Builder nodeBuilder, + long src, long dst, Label label) { + NodeDataMask nodeMask = new NodeDataMask(mask); + buildSuccessorProperties(graph, nodeMask, nodeBuilder, src, dst, label); + } +} diff --git a/java/src/main/java/org/softwareheritage/graph/rpc/Traversal.java b/java/src/main/java/org/softwareheritage/graph/rpc/Traversal.java index e7e9000..c036052 100644 --- a/java/src/main/java/org/softwareheritage/graph/rpc/Traversal.java +++ b/java/src/main/java/org/softwareheritage/graph/rpc/Traversal.java @@ -1,275 +1,178 @@ package org.softwareheritage.graph.rpc; -import com.google.protobuf.ByteString; import it.unimi.dsi.big.webgraph.LazyLongIterator; import it.unimi.dsi.big.webgraph.labelling.ArcLabelledNodeIterator; import it.unimi.dsi.big.webgraph.labelling.Label; import org.softwareheritage.graph.*; -import org.softwareheritage.graph.labels.DirEntry; import java.util.*; public class Traversal { private static LazyLongIterator filterSuccessors(SwhUnidirectionalGraph g, long nodeId, AllowedEdges allowedEdges) { if (allowedEdges.restrictedTo == null) { // All edges are allowed, bypass edge check return g.successors(nodeId); } else { LazyLongIterator allSuccessors = g.successors(nodeId); return new LazyLongIterator() { @Override public long nextLong() { long neighbor; while ((neighbor = allSuccessors.nextLong()) != -1) { if (allowedEdges.isAllowed(g.getNodeType(nodeId), g.getNodeType(neighbor))) { return neighbor; } } return -1; } @Override public long skip(final long n) { long i = 0; while (i < n && nextLong() != -1) i++; return i; } }; } } private static ArcLabelledNodeIterator.LabelledArcIterator filterLabelledSuccessors(SwhUnidirectionalGraph g, long nodeId, AllowedEdges allowedEdges) { if (allowedEdges.restrictedTo == null) { // All edges are allowed, bypass edge check return g.labelledSuccessors(nodeId); } else { ArcLabelledNodeIterator.LabelledArcIterator allSuccessors = g.labelledSuccessors(nodeId); return new ArcLabelledNodeIterator.LabelledArcIterator() { @Override public Label label() { return allSuccessors.label(); } @Override public long nextLong() { long neighbor; while ((neighbor = allSuccessors.nextLong()) != -1) { if (allowedEdges.isAllowed(g.getNodeType(nodeId), g.getNodeType(neighbor))) { return neighbor; } } return -1; } @Override public long skip(final long n) { long i = 0; while (i < n && nextLong() != -1) i++; return i; } }; } } private static class NodeFilterChecker { private final SwhUnidirectionalGraph g; private final NodeFilter filter; private final AllowedNodes allowedNodes; private NodeFilterChecker(SwhUnidirectionalGraph graph, NodeFilter filter) { this.g = graph; this.filter = filter; this.allowedNodes = new AllowedNodes(filter.hasTypes() ? filter.getTypes() : "*"); } public boolean allowed(long nodeId) { if (filter == null) { return true; } if (!this.allowedNodes.isAllowed(g.getNodeType(nodeId))) { return false; } return true; } } public static SwhUnidirectionalGraph getDirectedGraph(SwhBidirectionalGraph g, TraversalRequest request) { switch (request.getDirection()) { case FORWARD: return g.getForwardGraph(); case BACKWARD: return g.getBackwardGraph(); case BOTH: return new SwhUnidirectionalGraph(g.symmetrize(), g.getProperties()); } throw new IllegalArgumentException("Unknown direction: " + request.getDirection()); } public static void simpleTraversal(SwhBidirectionalGraph bidirectionalGraph, TraversalRequest request, NodeObserver nodeObserver) { SwhUnidirectionalGraph g = getDirectedGraph(bidirectionalGraph, request); NodeFilterChecker nodeReturnChecker = new NodeFilterChecker(g, request.getReturnNodes()); + NodePropertyBuilder.NodeDataMask nodeDataMask = new NodePropertyBuilder.NodeDataMask( + request.hasMask() ? request.getMask() : null); AllowedEdges allowedEdges = new AllowedEdges(request.hasEdges() ? request.getEdges() : "*"); Queue queue = new ArrayDeque<>(); HashSet visited = new HashSet<>(); request.getSrcList().forEach(srcSwhid -> { long srcNodeId = g.getNodeId(new SWHID(srcSwhid)); queue.add(srcNodeId); visited.add(srcNodeId); }); queue.add(-1L); // Depth sentinel long edgesAccessed = 0; long currentDepth = 0; while (!queue.isEmpty()) { long curr = queue.poll(); if (curr == -1L) { ++currentDepth; if (!queue.isEmpty()) { queue.add(-1L); } continue; } if (request.hasMaxDepth() && currentDepth > request.getMaxDepth()) { break; } edgesAccessed += g.outdegree(curr); if (request.hasMaxEdges() && edgesAccessed >= request.getMaxEdges()) { break; } Node.Builder nodeBuilder = null; if (nodeReturnChecker.allowed(curr) && (!request.hasMinDepth() || currentDepth >= request.getMinDepth())) { nodeBuilder = Node.newBuilder(); - buildNodeProperties(g, request.getReturnFields(), nodeBuilder, curr); + NodePropertyBuilder.buildNodeProperties(g, nodeDataMask, nodeBuilder, curr); } ArcLabelledNodeIterator.LabelledArcIterator it = filterLabelledSuccessors(g, curr, allowedEdges); long traversalSuccessors = 0; for (long succ; (succ = it.nextLong()) != -1;) { traversalSuccessors++; if (!visited.contains(succ)) { queue.add(succ); visited.add(succ); } - buildSuccessorProperties(g, request.getReturnFields(), nodeBuilder, curr, succ, it.label()); + NodePropertyBuilder.buildSuccessorProperties(g, nodeDataMask, nodeBuilder, curr, succ, it.label()); } if (request.getReturnNodes().hasMinTraversalSuccessors() && traversalSuccessors < request.getReturnNodes().getMinTraversalSuccessors() || request.getReturnNodes().hasMaxTraversalSuccessors() && traversalSuccessors > request.getReturnNodes().getMaxTraversalSuccessors()) { nodeBuilder = null; } if (nodeBuilder != null) { nodeObserver.onNext(nodeBuilder.build()); } } } - private static void buildNodeProperties(SwhUnidirectionalGraph graph, NodeFields fields, Node.Builder nodeBuilder, - long node) { - if (fields == null || !fields.hasSwhid() || fields.getSwhid()) { - nodeBuilder.setSwhid(graph.getSWHID(node).toString()); - } - if (fields == null) { - return; - } - - switch (graph.getNodeType(node)) { - case CNT: - if (fields.hasCntLength()) { - nodeBuilder.setCntLength(graph.getContentLength(node)); - } - if (fields.hasCntIsSkipped()) { - nodeBuilder.setCntIsSkipped(graph.isContentSkipped(node)); - } - break; - case REV: - if (fields.getRevAuthor()) { - nodeBuilder.setRevAuthor(graph.getAuthorId(node)); - } - if (fields.getRevCommitter()) { - nodeBuilder.setRevAuthor(graph.getCommitterId(node)); - } - if (fields.getRevAuthorDate()) { - nodeBuilder.setRevAuthorDate(graph.getAuthorTimestamp(node)); - } - if (fields.getRevAuthorDateOffset()) { - nodeBuilder.setRevAuthorDateOffset(graph.getAuthorTimestampOffset(node)); - } - if (fields.getRevCommitterDate()) { - nodeBuilder.setRevCommitterDate(graph.getCommitterTimestamp(node)); - } - if (fields.getRevCommitterDateOffset()) { - nodeBuilder.setRevCommitterDateOffset(graph.getCommitterTimestampOffset(node)); - } - if (fields.getRevMessage()) { - byte[] msg = graph.getMessage(node); - if (msg != null) { - nodeBuilder.setRevMessage(ByteString.copyFrom(msg)); - } - } - break; - case REL: - if (fields.getRelAuthor()) { - nodeBuilder.setRelAuthor(graph.getAuthorId(node)); - } - if (fields.getRelAuthorDate()) { - nodeBuilder.setRelAuthorDate(graph.getAuthorTimestamp(node)); - } - if (fields.getRelAuthorDateOffset()) { - nodeBuilder.setRelAuthorDateOffset(graph.getAuthorTimestampOffset(node)); - } - if (fields.getRelName()) { - byte[] msg = graph.getTagName(node); - if (msg != null) { - nodeBuilder.setRelName(ByteString.copyFrom(msg)); - } - } - if (fields.getRelMessage()) { - byte[] msg = graph.getMessage(node); - if (msg != null) { - nodeBuilder.setRelMessage(ByteString.copyFrom(msg)); - } - } - break; - case ORI: - if (fields.getOriUrl()) { - String url = graph.getUrl(node); - if (url != null) { - nodeBuilder.setOriUrl(url); - } - } - } - } - - private static void buildSuccessorProperties(SwhUnidirectionalGraph graph, NodeFields fields, - Node.Builder nodeBuilder, long src, long dst, Label label) { - if (nodeBuilder != null && fields != null && fields.getSuccessor()) { - Successor.Builder successorBuilder = Successor.newBuilder(); - if (!fields.hasSuccessorSwhid() || fields.getSuccessorSwhid()) { - successorBuilder.setSwhid(graph.getSWHID(dst).toString()); - } - if (fields.getSuccessorLabel()) { - DirEntry[] entries = (DirEntry[]) label.get(); - for (DirEntry entry : entries) { - EdgeLabel.Builder builder = EdgeLabel.newBuilder(); - builder.setName(ByteString.copyFrom(graph.getLabelName(entry.filenameId))); - builder.setPermission(entry.permission); - successorBuilder.addLabel(builder.build()); - } - } - nodeBuilder.addSuccessor(successorBuilder.build()); - } - } - public interface NodeObserver { void onNext(Node nodeId); } } diff --git a/java/src/test/java/org/softwareheritage/graph/rpc/GetNodeTest.java b/java/src/test/java/org/softwareheritage/graph/rpc/GetNodeTest.java new file mode 100644 index 0000000..a57fee0 --- /dev/null +++ b/java/src/test/java/org/softwareheritage/graph/rpc/GetNodeTest.java @@ -0,0 +1,279 @@ +package org.softwareheritage.graph.rpc; + +import com.google.protobuf.Descriptors; +import com.google.protobuf.FieldMask; +import io.grpc.StatusRuntimeException; +import org.junit.jupiter.api.Test; +import org.softwareheritage.graph.SWHID; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +public class GetNodeTest extends TraversalServiceTest { + @Test + public void testNotFound() { + assertThrows(StatusRuntimeException.class, + () -> client.getNode(GetNodeRequest.newBuilder().setSwhid(fakeSWHID("cnt", 404).toString()).build())); + } + + @Test + public void testInvalidSwhid() { + assertThrows(StatusRuntimeException.class, () -> client.getNode( + GetNodeRequest.newBuilder().setSwhid("swh:1:lol:0000000000000000000000000000000000000001").build())); + assertThrows(StatusRuntimeException.class, () -> client.getNode( + GetNodeRequest.newBuilder().setSwhid("swh:1:cnt:000000000000000000000000000000000000000z").build())); + } + + @Test + public void testContents() { + List expectedCnts = List.of(1, 4, 5, 7, 11, 14, 15); + Map expectedLengths = Map.of(1, 42, 4, 404, 5, 1337, 7, 666, 11, 313, 14, 14, 15, 404); + Set expectedSkipped = Set.of(15); + + for (Integer cntId : expectedCnts) { + Node n = client.getNode(GetNodeRequest.newBuilder().setSwhid(fakeSWHID("cnt", cntId).toString()).build()); + assertTrue(n.hasCnt()); + assertTrue(n.getCnt().hasLength()); + assertEquals((long) expectedLengths.get(cntId), n.getCnt().getLength()); + assertTrue(n.getCnt().hasIsSkipped()); + assertEquals(expectedSkipped.contains(cntId), n.getCnt().getIsSkipped()); + } + } + + @Test + public void testRevisions() { + List expectedRevs = List.of(3, 9, 13, 18); + Map expectedMessages = Map.of(3, "Initial commit", 9, "Add parser", 13, "Add tests", 18, + "Refactor codebase"); + + Map expectedAuthors = Map.of(3, "foo", 9, "bar", 13, "foo", 18, "baz"); + Map expectedCommitters = Map.of(3, "foo", 9, "bar", 13, "bar", 18, "foo"); + + Map expectedAuthorTimestamps = Map.of(3, 1111122220L, 9, 1111144440L, 13, 1111166660L, 18, + 1111177770L); + Map expectedCommitterTimestamps = Map.of(3, 1111122220L, 9, 1111155550L, 13, 1111166660L, 18, + 1111177770L); + Map expectedAuthorTimestampOffsets = Map.of(3, 120, 9, 120, 13, 120, 18, 0); + Map expectedCommitterTimestampOffsets = Map.of(3, 120, 9, 120, 13, 120, 18, 0); + + HashMap personMapping = new HashMap<>(); + for (Integer revId : expectedRevs) { + Node n = client.getNode(GetNodeRequest.newBuilder().setSwhid(fakeSWHID("rev", revId).toString()).build()); + assertTrue(n.hasRev()); + assertTrue(n.getRev().hasMessage()); + assertEquals(expectedMessages.get(revId), n.getRev().getMessage().toStringUtf8()); + + // Persons are anonymized, we just need to check that the mapping is self-consistent + assertTrue(n.getRev().hasAuthor()); + assertTrue(n.getRev().hasCommitter()); + int[] actualPersons = new int[]{(int) n.getRev().getAuthor(), (int) n.getRev().getCommitter()}; + String[] expectedPersons = new String[]{expectedAuthors.get(revId), expectedCommitters.get(revId)}; + for (int i = 0; i < actualPersons.length; i++) { + int actualPerson = actualPersons[i]; + String expectedPerson = expectedPersons[i]; + assertTrue(actualPerson >= 0); + if (personMapping.containsKey(actualPerson)) { + assertEquals(personMapping.get(actualPerson), expectedPerson); + } else { + personMapping.put(actualPerson, expectedPerson); + } + } + + assertTrue(n.getRev().hasAuthorDate()); + assertTrue(n.getRev().hasAuthorDateOffset()); + assertTrue(n.getRev().hasCommitterDate()); + assertTrue(n.getRev().hasCommitterDateOffset()); + + // FIXME: all the timestamps are one hour off?! + // System.err.println(revId + " " + n.getRev().getAuthorDate() + " " + + // n.getRev().getAuthorDateOffset()); + // System.err.println(revId + " " + n.getRev().getCommitterDate() + " " + + // n.getRev().getCommitterDateOffset()); + + // assertEquals(expectedAuthorTimestamps.get(revId), n.getRev().getAuthorDate()); + assertEquals(expectedAuthorTimestampOffsets.get(revId), n.getRev().getAuthorDateOffset()); + // assertEquals(expectedCommitterTimestamps.get(revId), n.getRev().getAuthorDate()); + assertEquals(expectedCommitterTimestampOffsets.get(revId), n.getRev().getAuthorDateOffset()); + } + } + + @Test + public void testReleases() { + List expectedRels = List.of(10, 19); + Map expectedMessages = Map.of(10, "Version 1.0", 19, "Version 2.0"); + Map expectedNames = Map.of(10, "v1.0", 19, "v2.0"); + + Map expectedAuthors = Map.of(10, "foo", 19, "bar"); + + Map expectedAuthorTimestamps = Map.of(10, 1234567890L); + Map expectedAuthorTimestampOffsets = Map.of(3, 120); + + HashMap personMapping = new HashMap<>(); + for (Integer relId : expectedRels) { + Node n = client.getNode(GetNodeRequest.newBuilder().setSwhid(fakeSWHID("rel", relId).toString()).build()); + assertTrue(n.hasRel()); + assertTrue(n.getRel().hasMessage()); + assertEquals(expectedMessages.get(relId), n.getRel().getMessage().toStringUtf8()); + // FIXME: names are always empty?! + // System.err.println(relId + " " + n.getRel().getName()); + // assertEquals(expectedNames.get(relId), n.getRel().getName().toStringUtf8()); + + // Persons are anonymized, we just need to check that the mapping is self-consistent + assertTrue(n.getRel().hasAuthor()); + int actualPerson = (int) n.getRel().getAuthor(); + String expectedPerson = expectedAuthors.get(relId); + assertTrue(actualPerson >= 0); + if (personMapping.containsKey(actualPerson)) { + assertEquals(personMapping.get(actualPerson), expectedPerson); + } else { + personMapping.put(actualPerson, expectedPerson); + } + + assertTrue(n.getRel().hasAuthorDate()); + assertTrue(n.getRel().hasAuthorDateOffset()); + + // FIXME: all the timestamps are one hour off?! + // if (expectedAuthorTimestamps.containsKey(relId)) { + // assertEquals(expectedAuthorTimestamps.get(revId), n.getRev().getAuthorDate()); + // } + if (expectedAuthorTimestampOffsets.containsKey(relId)) { + assertEquals(expectedAuthorTimestampOffsets.get(relId), n.getRev().getAuthorDateOffset()); + } + } + } + + @Test + public void testOrigins() { + List expectedOris = List.of(new SWHID(TEST_ORIGIN_ID)); + Map expectedUrls = Map.of(new SWHID(TEST_ORIGIN_ID), "https://example.com/swh/graph"); + + for (SWHID oriSwhid : expectedOris) { + Node n = client.getNode(GetNodeRequest.newBuilder().setSwhid(oriSwhid.toString()).build()); + assertTrue(n.hasOri()); + assertTrue(n.getOri().hasUrl()); + assertEquals(expectedUrls.get(oriSwhid), n.getOri().getUrl()); + } + } + + @Test + public void testCntMask() { + Node n; + String swhid = fakeSWHID("cnt", 1).toString(); + + // No mask, all fields present + n = client.getNode(GetNodeRequest.newBuilder().setSwhid(swhid).build()); + assertTrue(n.hasCnt()); + assertTrue(n.getCnt().hasLength()); + assertEquals(42, n.getCnt().getLength()); + assertTrue(n.getCnt().hasIsSkipped()); + assertFalse(n.getCnt().getIsSkipped()); + + // Empty mask, no fields present + n = client.getNode(GetNodeRequest.newBuilder().setSwhid(swhid).setMask(FieldMask.getDefaultInstance()).build()); + assertFalse(n.getCnt().hasLength()); + assertFalse(n.getCnt().hasIsSkipped()); + + // Mask with length, no isSkipped + n = client.getNode(GetNodeRequest.newBuilder().setSwhid(swhid) + .setMask(FieldMask.newBuilder().addPaths("cnt.length").build()).build()); + assertTrue(n.getCnt().hasLength()); + assertFalse(n.getCnt().hasIsSkipped()); + + // Mask with isSkipped, no length + n = client.getNode(GetNodeRequest.newBuilder().setSwhid(swhid) + .setMask(FieldMask.newBuilder().addPaths("cnt.is_skipped").build()).build()); + assertFalse(n.getCnt().hasLength()); + assertTrue(n.getCnt().hasIsSkipped()); + } + + @Test + public void testRevMask() { + Node n; + String swhid = fakeSWHID("rev", 3).toString(); + + // No mask, all fields present + n = client.getNode(GetNodeRequest.newBuilder().setSwhid(swhid).build()); + assertTrue(n.hasRev()); + assertTrue(n.getRev().hasMessage()); + assertTrue(n.getRev().hasAuthor()); + assertTrue(n.getRev().hasAuthorDate()); + assertTrue(n.getRev().hasAuthorDateOffset()); + assertTrue(n.getRev().hasCommitter()); + assertTrue(n.getRev().hasCommitterDate()); + assertTrue(n.getRev().hasCommitterDateOffset()); + + // Empty mask, no fields present + n = client.getNode(GetNodeRequest.newBuilder().setSwhid(swhid).setMask(FieldMask.getDefaultInstance()).build()); + assertFalse(n.getRev().hasMessage()); + assertFalse(n.getRev().hasAuthor()); + assertFalse(n.getRev().hasAuthorDate()); + assertFalse(n.getRev().hasAuthorDateOffset()); + assertFalse(n.getRev().hasCommitter()); + assertFalse(n.getRev().hasCommitterDate()); + assertFalse(n.getRev().hasCommitterDateOffset()); + + // Test all masks with single fields + for (Descriptors.FieldDescriptor includedField : RevisionData.getDefaultInstance().getAllFields().keySet()) { + n = client.getNode(GetNodeRequest.newBuilder().setSwhid(swhid) + .setMask(FieldMask.newBuilder().addPaths("rev." + includedField.getName()).build()).build()); + for (Descriptors.FieldDescriptor f : n.getRev().getDescriptorForType().getFields()) { + assertEquals(n.getRev().hasField(f), f.getName().equals(includedField.getName())); + } + } + } + + @Test + public void testRelMask() { + Node n; + String swhid = fakeSWHID("rel", 19).toString(); + + // No mask, all fields present + n = client.getNode(GetNodeRequest.newBuilder().setSwhid(swhid).build()); + assertTrue(n.hasRel()); + assertTrue(n.getRel().hasMessage()); + assertTrue(n.getRel().hasAuthor()); + assertTrue(n.getRel().hasAuthorDate()); + assertTrue(n.getRel().hasAuthorDateOffset()); + + // Empty mask, no fields present + n = client.getNode(GetNodeRequest.newBuilder().setSwhid(swhid).setMask(FieldMask.getDefaultInstance()).build()); + assertFalse(n.getRel().hasMessage()); + assertFalse(n.getRel().hasAuthor()); + assertFalse(n.getRel().hasAuthorDate()); + assertFalse(n.getRel().hasAuthorDateOffset()); + + // Test all masks with single fields + for (Descriptors.FieldDescriptor includedField : ReleaseData.getDefaultInstance().getAllFields().keySet()) { + n = client.getNode(GetNodeRequest.newBuilder().setSwhid(swhid) + .setMask(FieldMask.newBuilder().addPaths("rel." + includedField.getName()).build()).build()); + for (Descriptors.FieldDescriptor f : n.getRel().getDescriptorForType().getFields()) { + assertEquals(n.getRel().hasField(f), f.getName().equals(includedField.getName())); + } + } + } + + @Test + public void testOriMask() { + Node n; + String swhid = TEST_ORIGIN_ID; + + // No mask, all fields present + n = client.getNode(GetNodeRequest.newBuilder().setSwhid(swhid).build()); + assertTrue(n.hasOri()); + assertTrue(n.getOri().hasUrl()); + + // Empty mask, no fields present + n = client.getNode(GetNodeRequest.newBuilder().setSwhid(swhid).setMask(FieldMask.getDefaultInstance()).build()); + assertFalse(n.getOri().hasUrl()); + + // Test all masks with single fields + for (Descriptors.FieldDescriptor includedField : OriginData.getDefaultInstance().getAllFields().keySet()) { + n = client.getNode(GetNodeRequest.newBuilder().setSwhid(swhid) + .setMask(FieldMask.newBuilder().addPaths("ori." + includedField.getName()).build()).build()); + for (Descriptors.FieldDescriptor f : n.getOri().getDescriptorForType().getFields()) { + assertEquals(n.getOri().hasField(f), f.getName().equals(includedField.getName())); + } + } + } +} diff --git a/java/src/test/java/org/softwareheritage/graph/rpc/TraversalServiceTest.java b/java/src/test/java/org/softwareheritage/graph/rpc/TraversalServiceTest.java index 464675a..f038327 100644 --- a/java/src/test/java/org/softwareheritage/graph/rpc/TraversalServiceTest.java +++ b/java/src/test/java/org/softwareheritage/graph/rpc/TraversalServiceTest.java @@ -1,48 +1,50 @@ package org.softwareheritage.graph.rpc; import io.grpc.ManagedChannel; import io.grpc.Server; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; import io.grpc.testing.GrpcCleanupRule; import org.junit.Rule; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.softwareheritage.graph.GraphTest; import org.softwareheritage.graph.SWHID; +import org.softwareheritage.graph.SwhBidirectionalGraph; import java.util.ArrayList; import java.util.Iterator; public class TraversalServiceTest extends GraphTest { @Rule public final GrpcCleanupRule grpcCleanup = new GrpcCleanupRule(); private static Server server; private static ManagedChannel channel; + protected static SwhBidirectionalGraph g; protected static TraversalServiceGrpc.TraversalServiceBlockingStub client; @BeforeAll static void setup() throws Exception { String serverName = InProcessServerBuilder.generateName(); - assert getGraph() != null; + g = GraphServer.loadGraph(getGraphPath().toString()); server = InProcessServerBuilder.forName(serverName).directExecutor() - .addService(new GraphServer.TraversalService(getGraph())).build().start(); + .addService(new GraphServer.TraversalService(g.copy())).build().start(); channel = InProcessChannelBuilder.forName(serverName).directExecutor().build(); client = TraversalServiceGrpc.newBlockingStub(channel); } @AfterAll - static void teardown() throws Exception { + static void teardown() { channel.shutdownNow(); server.shutdownNow(); } public ArrayList getSWHIDs(Iterator it) { - ArrayList actualLeaves = new ArrayList<>(); + ArrayList res = new ArrayList<>(); it.forEachRemaining((Node n) -> { - actualLeaves.add(new SWHID(n.getSwhid())); + res.add(new SWHID(n.getSwhid())); }); - return actualLeaves; + return res; } } diff --git a/java/src/test/java/org/softwareheritage/graph/rpc/TraverseNodesPropertiesTest.java b/java/src/test/java/org/softwareheritage/graph/rpc/TraverseNodesPropertiesTest.java new file mode 100644 index 0000000..2c55507 --- /dev/null +++ b/java/src/test/java/org/softwareheritage/graph/rpc/TraverseNodesPropertiesTest.java @@ -0,0 +1,110 @@ +package org.softwareheritage.graph.rpc; + +import com.google.protobuf.Descriptors; +import com.google.protobuf.FieldMask; +import com.google.protobuf.Message; +import it.unimi.dsi.big.webgraph.labelling.ArcLabelledNodeIterator; +import org.junit.jupiter.api.Test; +import org.softwareheritage.graph.SWHID; +import org.softwareheritage.graph.SwhUnidirectionalGraph; +import org.softwareheritage.graph.labels.DirEntry; + +import java.util.*; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TraverseNodesPropertiesTest extends TraversalServiceTest { + private TraversalRequest.Builder getTraversalRequestBuilder(SWHID src) { + return TraversalRequest.newBuilder().addSrc(src.toString()); + } + + private void checkHasAllFields(Message m) { + for (Descriptors.FieldDescriptor fd : m.getAllFields().keySet()) { + assertTrue(m.hasField(fd)); + } + } + + private void checkHasAllFieldsOfType(Node node) { + if (node.hasCnt()) { + checkHasAllFields(node.getCnt()); + } + if (node.hasRev()) { + checkHasAllFields(node.getRev()); + } + if (node.hasRel()) { + checkHasAllFields(node.getRel()); + } + if (node.hasOri()) { + checkHasAllFields(node.getOri()); + } + } + + private void checkSuccessors(SwhUnidirectionalGraph g, Node node) { + HashMap graphSuccessors = new HashMap<>(); + ArcLabelledNodeIterator.LabelledArcIterator it = g.labelledSuccessors(g.getNodeId(new SWHID(node.getSwhid()))); + long succ; + while ((succ = it.nextLong()) != -1) { + graphSuccessors.put(g.getSWHID(succ).toString(), (DirEntry[]) it.label().get()); + } + + assertEquals(node.getSuccessorList().stream().map(Successor::getSwhid).collect(Collectors.toSet()), + graphSuccessors.keySet()); + + for (Successor successor : node.getSuccessorList()) { + DirEntry[] expectedArray = graphSuccessors.get(successor.getSwhid()); + HashMap expectedLabels = new HashMap<>(); + for (DirEntry dirEntry : expectedArray) { + expectedLabels.put(new String(g.getLabelName(dirEntry.filenameId)), dirEntry.permission); + } + for (EdgeLabel edgeLabel : successor.getLabelList()) { + assertTrue(expectedLabels.containsKey(edgeLabel.getName().toStringUtf8())); + if (edgeLabel.getPermission() > 0) { + assertEquals(edgeLabel.getPermission(), expectedLabels.get(edgeLabel.getName().toStringUtf8())); + } + } + } + } + + @Test + public void forwardFromRoot() { + ArrayList response = new ArrayList<>(); + client.traverse(getTraversalRequestBuilder(new SWHID(TEST_ORIGIN_ID)).build()).forEachRemaining(response::add); + for (Node node : response) { + checkHasAllFieldsOfType(node); + checkSuccessors(g.getForwardGraph(), node); + } + } + + @Test + public void backwardFromLeaf() { + ArrayList response = new ArrayList<>(); + client.traverse(getTraversalRequestBuilder(fakeSWHID("cnt", 4)).setDirection(GraphDirection.BACKWARD).build()) + .forEachRemaining(response::add); + for (Node node : response) { + checkHasAllFieldsOfType(node); + checkSuccessors(g.getBackwardGraph(), node); + } + } + + @Test + public void forwardFromRootMaskedLabels() { + ArrayList response = new ArrayList<>(); + client.traverse(getTraversalRequestBuilder(new SWHID(TEST_ORIGIN_ID)) + .setMask(FieldMask.newBuilder().addPaths("successor.swhid").addPaths("swhid").build()).build()) + .forEachRemaining(response::add); + for (Node node : response) { + HashSet graphSuccessors = new HashSet<>(); + ArcLabelledNodeIterator.LabelledArcIterator it = g + .labelledSuccessors(g.getNodeId(new SWHID(node.getSwhid()))); + long succ; + while ((succ = it.nextLong()) != -1) { + graphSuccessors.add(g.getSWHID(succ).toString()); + } + + assertEquals(node.getSuccessorList().stream().map(Successor::getSwhid).collect(Collectors.toSet()), + graphSuccessors); + } + } +} diff --git a/proto/swhgraph.proto b/proto/swhgraph.proto index db88a97..7637941 100644 --- a/proto/swhgraph.proto +++ b/proto/swhgraph.proto @@ -1,134 +1,129 @@ syntax = "proto3"; +import "google/protobuf/field_mask.proto"; + option java_multiple_files = true; option java_package = "org.softwareheritage.graph.rpc"; option java_outer_classname = "GraphService"; package swh.graph; service TraversalService { rpc Traverse (TraversalRequest) returns (stream Node); rpc CountNodes (TraversalRequest) returns (CountResponse); rpc CountEdges (TraversalRequest) returns (CountResponse); rpc Stats (StatsRequest) returns (StatsResponse); rpc CheckSwhid (CheckSwhidRequest) returns (CheckSwhidResponse); + rpc GetNode (GetNodeRequest) returns (Node); } enum GraphDirection { FORWARD = 0; BACKWARD = 1; BOTH = 2; } message TraversalRequest { repeated string src = 1; // Traversal options - optional GraphDirection direction = 2; + GraphDirection direction = 2; optional string edges = 3; optional int64 max_edges = 4; optional int64 min_depth = 5; optional int64 max_depth = 6; optional NodeFilter return_nodes = 7; - optional NodeFields return_fields = 8; + optional google.protobuf.FieldMask mask = 8; } message NodeFilter { optional string types = 1; optional int64 min_traversal_successors = 2; optional int64 max_traversal_successors = 3; } -message NodeFields { - optional bool swhid = 1; - - optional bool successor = 2; - optional bool successor_swhid = 3; - optional bool successor_label = 4; - - optional bool cnt_length = 5; - optional bool cnt_is_skipped = 6; - - optional bool rev_author = 7; - optional bool rev_author_date = 8; - optional bool rev_author_date_offset = 9; - optional bool rev_committer = 10; - optional bool rev_committer_date = 11; - optional bool rev_committer_date_offset = 12; - optional bool rev_message = 13; - - optional bool rel_author = 14; - optional bool rel_author_date = 15; - optional bool rel_author_date_offset = 16; - optional bool rel_name = 17; - optional bool rel_message = 18; - - optional bool ori_url = 19; -} - message Node { string swhid = 1; repeated Successor successor = 2; + oneof data { + ContentData cnt = 3; + RevisionData rev = 5; + ReleaseData rel = 6; + OriginData ori = 8; + }; +} - optional int64 cnt_length = 3; - optional bool cnt_is_skipped = 4; +message Successor { + optional string swhid = 1; + repeated EdgeLabel label = 2; +} - optional int64 rev_author = 5; - optional int64 rev_author_date = 6; - optional int32 rev_author_date_offset = 7; - optional int64 rev_committer = 8; - optional int64 rev_committer_date = 9; - optional int32 rev_committer_date_offset = 10; - optional bytes rev_message = 11; +message ContentData { + optional int64 length = 1; + optional bool is_skipped = 2; +} - optional int64 rel_author = 12; - optional int64 rel_author_date = 13; - optional int32 rel_author_date_offset = 14; - optional bytes rel_name = 15; - optional bytes rel_message = 16; +message RevisionData { + optional int64 author = 1; + optional int64 author_date = 2; + optional int32 author_date_offset = 3; + optional int64 committer = 4; + optional int64 committer_date = 5; + optional int32 committer_date_offset = 6; + optional bytes message = 7; +} - optional string ori_url = 17; +message ReleaseData { + optional int64 author = 1; + optional int64 author_date = 2; + optional int32 author_date_offset = 3; + optional bytes name = 4; + optional bytes message = 5; } -message Successor { - optional string swhid = 1; - repeated EdgeLabel label = 2; +message OriginData { + optional string url = 1; } message EdgeLabel { bytes name = 1; int32 permission = 2; } message CountResponse { int64 count = 1; } message StatsRequest { } message StatsResponse { int64 num_nodes = 1; int64 num_edges = 2; double compression = 3; double bits_per_node = 4; double bits_per_edge = 5; double avg_locality = 6; int64 indegree_min = 7; int64 indegree_max = 8; double indegree_avg = 9; int64 outdegree_min = 10; int64 outdegree_max = 11; double outdegree_avg = 12; } message CheckSwhidRequest { string swhid = 1; } +message GetNodeRequest { + string swhid = 1; + optional google.protobuf.FieldMask mask = 8; +} + message CheckSwhidResponse { bool exists = 1; string details = 2; } diff --git a/swh/graph/http_server.py b/swh/graph/http_server.py index 8a9bbc6..69a70f1 100644 --- a/swh/graph/http_server.py +++ b/swh/graph/http_server.py @@ -1,344 +1,346 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """ A proxy HTTP server for swh-graph, talking to the Java code via py4j, and using FIFO as a transport to stream integers between the two languages. """ import json import os from typing import Optional import aiohttp.test_utils import aiohttp.web from google.protobuf import json_format +from google.protobuf.field_mask_pb2 import FieldMask import grpc from swh.core.api.asynchronous import RPCServerApp from swh.core.config import read as config_read from swh.graph.rpc.swhgraph_pb2 import ( CheckSwhidRequest, - NodeFields, NodeFilter, StatsRequest, TraversalRequest, ) from swh.graph.rpc.swhgraph_pb2_grpc import TraversalServiceStub from swh.graph.rpc_server import spawn_java_rpc_server from swh.model.swhids import EXTENDED_SWHID_TYPES try: from contextlib import asynccontextmanager except ImportError: # Compatibility with 3.6 backport from async_generator import asynccontextmanager # type: ignore # maximum number of retries for random walks RANDOM_RETRIES = 10 # TODO make this configurable via rpc-serve configuration class GraphServerApp(RPCServerApp): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.on_startup.append(self._start) self.on_shutdown.append(self._stop) @staticmethod async def _start(app): app["channel"] = grpc.aio.insecure_channel(app["rpc_url"]) await app["channel"].__aenter__() app["rpc_client"] = TraversalServiceStub(app["channel"]) await app["rpc_client"].Stats(StatsRequest(), wait_for_ready=True) @staticmethod async def _stop(app): await app["channel"].__aexit__(None, None, None) if app.get("local_server"): app["local_server"].terminate() async def index(request): return aiohttp.web.Response( content_type="text/html", body=""" Software Heritage graph server

You have reached the Software Heritage graph API server.

See its API documentation for more information.

""", ) class GraphView(aiohttp.web.View): """Base class for views working on the graph, with utility functions""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.rpc_client: TraversalServiceStub = self.request.app["rpc_client"] def get_direction(self): """Validate HTTP query parameter `direction`""" s = self.request.query.get("direction", "forward") if s not in ("forward", "backward"): raise aiohttp.web.HTTPBadRequest(text=f"invalid direction: {s}") return s.upper() def get_edges(self): """Validate HTTP query parameter `edges`, i.e., edge restrictions""" s = self.request.query.get("edges", "*") if any( [ node_type != "*" and node_type not in EXTENDED_SWHID_TYPES for edge in s.split(":") for node_type in edge.split(",", maxsplit=1) ] ): raise aiohttp.web.HTTPBadRequest(text=f"invalid edge restriction: {s}") return s def get_return_types(self): """Validate HTTP query parameter 'return types', i.e, a set of types which we will filter the query results with""" s = self.request.query.get("return_types", "*") if any( node_type != "*" and node_type not in EXTENDED_SWHID_TYPES for node_type in s.split(",") ): raise aiohttp.web.HTTPBadRequest( text=f"invalid type for filtering res: {s}" ) # if the user puts a star, # then we filter nothing, we don't need the other information if "*" in s: return "*" else: return s def get_traversal(self): """Validate HTTP query parameter `traversal`, i.e., visit order""" s = self.request.query.get("traversal", "dfs") if s not in ("bfs", "dfs"): raise aiohttp.web.HTTPBadRequest(text=f"invalid traversal order: {s}") return s def get_limit(self): """Validate HTTP query parameter `limit`, i.e., number of results""" s = self.request.query.get("limit", "0") try: return int(s) except ValueError: raise aiohttp.web.HTTPBadRequest(text=f"invalid limit value: {s}") def get_max_edges(self): """Validate HTTP query parameter 'max_edges', i.e., the limit of the number of edges that can be visited""" s = self.request.query.get("max_edges", "0") try: return int(s) except ValueError: raise aiohttp.web.HTTPBadRequest(text=f"invalid max_edges value: {s}") async def check_swhid(self, swhid): """Validate that the given SWHID exists in the graph""" r = await self.rpc_client.CheckSwhid(CheckSwhidRequest(swhid=swhid)) if not r.exists: raise aiohttp.web.HTTPBadRequest(text=str(r.details)) class StreamingGraphView(GraphView): """Base class for views streaming their response line by line.""" content_type = "text/plain" @asynccontextmanager async def response_streamer(self, *args, **kwargs): """Context manager to prepare then close a StreamResponse""" response = aiohttp.web.StreamResponse(*args, **kwargs) response.content_type = self.content_type await response.prepare(self.request) yield response await response.write_eof() async def get(self): await self.prepare_response() async with self.response_streamer() as self.response_stream: self._buf = [] try: await self.stream_response() finally: await self._flush_buffer() return self.response_stream async def prepare_response(self): """This can be overridden with some setup to be run before the response actually starts streaming. """ pass async def stream_response(self): """Override this to perform the response streaming. Implementations of this should await self.stream_line(line) to write each line. """ raise NotImplementedError async def stream_line(self, line): """Write a line in the response stream.""" self._buf.append(line) if len(self._buf) > 100: await self._flush_buffer() async def _flush_buffer(self): await self.response_stream.write("\n".join(self._buf).encode() + b"\n") self._buf = [] class StatsView(GraphView): """View showing some statistics on the graph""" async def get(self): res = await self.rpc_client.Stats(StatsRequest()) stats = json_format.MessageToDict( res, including_default_value_fields=True, preserving_proto_field_name=True ) # Int64 fields are serialized as strings by default. for descriptor in res.DESCRIPTOR.fields: if descriptor.type == descriptor.TYPE_INT64: try: stats[descriptor.name] = int(stats[descriptor.name]) except KeyError: pass json_body = json.dumps(stats, indent=4, sort_keys=True) return aiohttp.web.Response(body=json_body, content_type="application/json") class SimpleTraversalView(StreamingGraphView): """Base class for views of simple traversals""" async def prepare_response(self): src = self.request.match_info["src"] self.traversal_request = TraversalRequest( src=[src], edges=self.get_edges(), direction=self.get_direction(), return_nodes=NodeFilter(types=self.get_return_types()), - return_fields=NodeFields(), + mask=FieldMask(paths=["swhid"]), ) if self.get_max_edges(): self.traversal_request.max_edges = self.get_max_edges() await self.check_swhid(src) self.configure_request() def configure_request(self): pass async def stream_response(self): async for node in self.rpc_client.Traverse(self.traversal_request): await self.stream_line(node.swhid) class LeavesView(SimpleTraversalView): def configure_request(self): self.traversal_request.return_nodes.max_traversal_successors = 0 class NeighborsView(SimpleTraversalView): def configure_request(self): self.traversal_request.min_depth = 1 self.traversal_request.max_depth = 1 class VisitNodesView(SimpleTraversalView): pass class VisitEdgesView(SimpleTraversalView): def configure_request(self): - self.traversal_request.return_fields.successor = True + self.traversal_request.mask.paths.extend(["successor", "successor.swhid"]) + # self.traversal_request.return_fields.successor = True async def stream_response(self): + print(self.traversal_request.mask) async for node in self.rpc_client.Traverse(self.traversal_request): for succ in node.successor: await self.stream_line(node.swhid + " " + succ.swhid) class CountView(GraphView): """Base class for counting views.""" count_type: Optional[str] = None async def get(self): src = self.request.match_info["src"] self.traversal_request = TraversalRequest( src=[src], edges=self.get_edges(), direction=self.get_direction(), return_nodes=NodeFilter(types=self.get_return_types()), - return_fields=NodeFields(), + mask=FieldMask(paths=["swhid"]), ) if self.get_max_edges(): self.traversal_request.max_edges = self.get_max_edges() self.configure_request() res = await self.rpc_client.CountNodes(self.traversal_request) return aiohttp.web.Response( body=str(res.count), content_type="application/json" ) def configure_request(self): pass class CountNeighborsView(CountView): def configure_request(self): self.traversal_request.min_depth = 1 self.traversal_request.max_depth = 1 class CountLeavesView(CountView): def configure_request(self): self.traversal_request.return_nodes.max_traversal_successors = 0 class CountVisitNodesView(CountView): pass def make_app(config=None, rpc_url=None, **kwargs): app = GraphServerApp(**kwargs) if rpc_url is None: app["local_server"], port = spawn_java_rpc_server(config) rpc_url = f"localhost:{port}" app.add_routes( [ aiohttp.web.get("/", index), aiohttp.web.get("/graph", index), aiohttp.web.view("/graph/stats", StatsView), aiohttp.web.view("/graph/leaves/{src}", LeavesView), aiohttp.web.view("/graph/neighbors/{src}", NeighborsView), aiohttp.web.view("/graph/visit/nodes/{src}", VisitNodesView), aiohttp.web.view("/graph/visit/edges/{src}", VisitEdgesView), aiohttp.web.view("/graph/neighbors/count/{src}", CountNeighborsView), aiohttp.web.view("/graph/leaves/count/{src}", CountLeavesView), aiohttp.web.view("/graph/visit/nodes/count/{src}", CountVisitNodesView), ] ) app["rpc_url"] = rpc_url return app def make_app_from_configfile(): """Load configuration and then build application to run""" config_file = os.environ.get("SWH_CONFIG_FILENAME") config = config_read(config_file) return make_app(config=config) diff --git a/swh/graph/rpc/swhgraph_pb2.py b/swh/graph/rpc/swhgraph_pb2.py index 2c77b08..3a16ac7 100644 --- a/swh/graph/rpc/swhgraph_pb2.py +++ b/swh/graph/rpc/swhgraph_pb2.py @@ -1,146 +1,187 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: swh/graph/rpc/swhgraph.proto """Generated protocol buffer code.""" from google.protobuf.internal import enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() +from google.protobuf import field_mask_pb2 as google_dot_protobuf_dot_field__mask__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cswh/graph/rpc/swhgraph.proto\x12\tswh.graph\"\xf8\x02\n\x10TraversalRequest\x12\x0b\n\x03src\x18\x01 \x03(\t\x12\x31\n\tdirection\x18\x02 \x01(\x0e\x32\x19.swh.graph.GraphDirectionH\x00\x88\x01\x01\x12\x12\n\x05\x65\x64ges\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tmax_edges\x18\x04 \x01(\x03H\x02\x88\x01\x01\x12\x16\n\tmin_depth\x18\x05 \x01(\x03H\x03\x88\x01\x01\x12\x16\n\tmax_depth\x18\x06 \x01(\x03H\x04\x88\x01\x01\x12\x30\n\x0creturn_nodes\x18\x07 \x01(\x0b\x32\x15.swh.graph.NodeFilterH\x05\x88\x01\x01\x12\x31\n\rreturn_fields\x18\x08 \x01(\x0b\x32\x15.swh.graph.NodeFieldsH\x06\x88\x01\x01\x42\x0c\n\n_directionB\x08\n\x06_edgesB\x0c\n\n_max_edgesB\x0c\n\n_min_depthB\x0c\n\n_max_depthB\x0f\n\r_return_nodesB\x10\n\x0e_return_fields\"\xb2\x01\n\nNodeFilter\x12\x12\n\x05types\x18\x01 \x01(\tH\x00\x88\x01\x01\x12%\n\x18min_traversal_successors\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12%\n\x18max_traversal_successors\x18\x03 \x01(\x03H\x02\x88\x01\x01\x42\x08\n\x06_typesB\x1b\n\x19_min_traversal_successorsB\x1b\n\x19_max_traversal_successors\"\x86\x07\n\nNodeFields\x12\x12\n\x05swhid\x18\x01 \x01(\x08H\x00\x88\x01\x01\x12\x16\n\tsuccessor\x18\x02 \x01(\x08H\x01\x88\x01\x01\x12\x1c\n\x0fsuccessor_swhid\x18\x03 \x01(\x08H\x02\x88\x01\x01\x12\x1c\n\x0fsuccessor_label\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x17\n\ncnt_length\x18\x05 \x01(\x08H\x04\x88\x01\x01\x12\x1b\n\x0e\x63nt_is_skipped\x18\x06 \x01(\x08H\x05\x88\x01\x01\x12\x17\n\nrev_author\x18\x07 \x01(\x08H\x06\x88\x01\x01\x12\x1c\n\x0frev_author_date\x18\x08 \x01(\x08H\x07\x88\x01\x01\x12#\n\x16rev_author_date_offset\x18\t \x01(\x08H\x08\x88\x01\x01\x12\x1a\n\rrev_committer\x18\n \x01(\x08H\t\x88\x01\x01\x12\x1f\n\x12rev_committer_date\x18\x0b \x01(\x08H\n\x88\x01\x01\x12&\n\x19rev_committer_date_offset\x18\x0c \x01(\x08H\x0b\x88\x01\x01\x12\x18\n\x0brev_message\x18\r \x01(\x08H\x0c\x88\x01\x01\x12\x17\n\nrel_author\x18\x0e \x01(\x08H\r\x88\x01\x01\x12\x1c\n\x0frel_author_date\x18\x0f \x01(\x08H\x0e\x88\x01\x01\x12#\n\x16rel_author_date_offset\x18\x10 \x01(\x08H\x0f\x88\x01\x01\x12\x15\n\x08rel_name\x18\x11 \x01(\x08H\x10\x88\x01\x01\x12\x18\n\x0brel_message\x18\x12 \x01(\x08H\x11\x88\x01\x01\x12\x14\n\x07ori_url\x18\x13 \x01(\x08H\x12\x88\x01\x01\x42\x08\n\x06_swhidB\x0c\n\n_successorB\x12\n\x10_successor_swhidB\x12\n\x10_successor_labelB\r\n\x0b_cnt_lengthB\x11\n\x0f_cnt_is_skippedB\r\n\x0b_rev_authorB\x12\n\x10_rev_author_dateB\x19\n\x17_rev_author_date_offsetB\x10\n\x0e_rev_committerB\x15\n\x13_rev_committer_dateB\x1c\n\x1a_rev_committer_date_offsetB\x0e\n\x0c_rev_messageB\r\n\x0b_rel_authorB\x12\n\x10_rel_author_dateB\x19\n\x17_rel_author_date_offsetB\x0b\n\t_rel_nameB\x0e\n\x0c_rel_messageB\n\n\x08_ori_url\"\x90\x06\n\x04Node\x12\r\n\x05swhid\x18\x01 \x01(\t\x12\'\n\tsuccessor\x18\x02 \x03(\x0b\x32\x14.swh.graph.Successor\x12\x17\n\ncnt_length\x18\x03 \x01(\x03H\x00\x88\x01\x01\x12\x1b\n\x0e\x63nt_is_skipped\x18\x04 \x01(\x08H\x01\x88\x01\x01\x12\x17\n\nrev_author\x18\x05 \x01(\x03H\x02\x88\x01\x01\x12\x1c\n\x0frev_author_date\x18\x06 \x01(\x03H\x03\x88\x01\x01\x12#\n\x16rev_author_date_offset\x18\x07 \x01(\x05H\x04\x88\x01\x01\x12\x1a\n\rrev_committer\x18\x08 \x01(\x03H\x05\x88\x01\x01\x12\x1f\n\x12rev_committer_date\x18\t \x01(\x03H\x06\x88\x01\x01\x12&\n\x19rev_committer_date_offset\x18\n \x01(\x05H\x07\x88\x01\x01\x12\x18\n\x0brev_message\x18\x0b \x01(\x0cH\x08\x88\x01\x01\x12\x17\n\nrel_author\x18\x0c \x01(\x03H\t\x88\x01\x01\x12\x1c\n\x0frel_author_date\x18\r \x01(\x03H\n\x88\x01\x01\x12#\n\x16rel_author_date_offset\x18\x0e \x01(\x05H\x0b\x88\x01\x01\x12\x15\n\x08rel_name\x18\x0f \x01(\x0cH\x0c\x88\x01\x01\x12\x18\n\x0brel_message\x18\x10 \x01(\x0cH\r\x88\x01\x01\x12\x14\n\x07ori_url\x18\x11 \x01(\tH\x0e\x88\x01\x01\x42\r\n\x0b_cnt_lengthB\x11\n\x0f_cnt_is_skippedB\r\n\x0b_rev_authorB\x12\n\x10_rev_author_dateB\x19\n\x17_rev_author_date_offsetB\x10\n\x0e_rev_committerB\x15\n\x13_rev_committer_dateB\x1c\n\x1a_rev_committer_date_offsetB\x0e\n\x0c_rev_messageB\r\n\x0b_rel_authorB\x12\n\x10_rel_author_dateB\x19\n\x17_rel_author_date_offsetB\x0b\n\t_rel_nameB\x0e\n\x0c_rel_messageB\n\n\x08_ori_url\"N\n\tSuccessor\x12\x12\n\x05swhid\x18\x01 \x01(\tH\x00\x88\x01\x01\x12#\n\x05label\x18\x02 \x03(\x0b\x32\x14.swh.graph.EdgeLabelB\x08\n\x06_swhid\"-\n\tEdgeLabel\x12\x0c\n\x04name\x18\x01 \x01(\x0c\x12\x12\n\npermission\x18\x02 \x01(\x05\"\x1e\n\rCountResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\"\x0e\n\x0cStatsRequest\"\x95\x02\n\rStatsResponse\x12\x11\n\tnum_nodes\x18\x01 \x01(\x03\x12\x11\n\tnum_edges\x18\x02 \x01(\x03\x12\x13\n\x0b\x63ompression\x18\x03 \x01(\x01\x12\x15\n\rbits_per_node\x18\x04 \x01(\x01\x12\x15\n\rbits_per_edge\x18\x05 \x01(\x01\x12\x14\n\x0c\x61vg_locality\x18\x06 \x01(\x01\x12\x14\n\x0cindegree_min\x18\x07 \x01(\x03\x12\x14\n\x0cindegree_max\x18\x08 \x01(\x03\x12\x14\n\x0cindegree_avg\x18\t \x01(\x01\x12\x15\n\routdegree_min\x18\n \x01(\x03\x12\x15\n\routdegree_max\x18\x0b \x01(\x03\x12\x15\n\routdegree_avg\x18\x0c \x01(\x01\"\"\n\x11\x43heckSwhidRequest\x12\r\n\x05swhid\x18\x01 \x01(\t\"5\n\x12\x43heckSwhidResponse\x12\x0e\n\x06\x65xists\x18\x01 \x01(\x08\x12\x0f\n\x07\x64\x65tails\x18\x02 \x01(\t*5\n\x0eGraphDirection\x12\x0b\n\x07\x46ORWARD\x10\x00\x12\x0c\n\x08\x42\x41\x43KWARD\x10\x01\x12\x08\n\x04\x42OTH\x10\x02\x32\xdf\x02\n\x10TraversalService\x12:\n\x08Traverse\x12\x1b.swh.graph.TraversalRequest\x1a\x0f.swh.graph.Node0\x01\x12\x43\n\nCountNodes\x12\x1b.swh.graph.TraversalRequest\x1a\x18.swh.graph.CountResponse\x12\x43\n\nCountEdges\x12\x1b.swh.graph.TraversalRequest\x1a\x18.swh.graph.CountResponse\x12:\n\x05Stats\x12\x17.swh.graph.StatsRequest\x1a\x18.swh.graph.StatsResponse\x12I\n\nCheckSwhid\x12\x1c.swh.graph.CheckSwhidRequest\x1a\x1d.swh.graph.CheckSwhidResponseB0\n\x1eorg.softwareheritage.graph.rpcB\x0cGraphServiceP\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cswh/graph/rpc/swhgraph.proto\x12\tswh.graph\x1a google/protobuf/field_mask.proto\"\xd8\x02\n\x10TraversalRequest\x12\x0b\n\x03src\x18\x01 \x03(\t\x12,\n\tdirection\x18\x02 \x01(\x0e\x32\x19.swh.graph.GraphDirection\x12\x12\n\x05\x65\x64ges\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tmax_edges\x18\x04 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\tmin_depth\x18\x05 \x01(\x03H\x02\x88\x01\x01\x12\x16\n\tmax_depth\x18\x06 \x01(\x03H\x03\x88\x01\x01\x12\x30\n\x0creturn_nodes\x18\x07 \x01(\x0b\x32\x15.swh.graph.NodeFilterH\x04\x88\x01\x01\x12-\n\x04mask\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x05\x88\x01\x01\x42\x08\n\x06_edgesB\x0c\n\n_max_edgesB\x0c\n\n_min_depthB\x0c\n\n_max_depthB\x0f\n\r_return_nodesB\x07\n\x05_mask\"\xb2\x01\n\nNodeFilter\x12\x12\n\x05types\x18\x01 \x01(\tH\x00\x88\x01\x01\x12%\n\x18min_traversal_successors\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12%\n\x18max_traversal_successors\x18\x03 \x01(\x03H\x02\x88\x01\x01\x42\x08\n\x06_typesB\x1b\n\x19_min_traversal_successorsB\x1b\n\x19_max_traversal_successors\"\xe2\x01\n\x04Node\x12\r\n\x05swhid\x18\x01 \x01(\t\x12\'\n\tsuccessor\x18\x02 \x03(\x0b\x32\x14.swh.graph.Successor\x12%\n\x03\x63nt\x18\x03 \x01(\x0b\x32\x16.swh.graph.ContentDataH\x00\x12&\n\x03rev\x18\x05 \x01(\x0b\x32\x17.swh.graph.RevisionDataH\x00\x12%\n\x03rel\x18\x06 \x01(\x0b\x32\x16.swh.graph.ReleaseDataH\x00\x12$\n\x03ori\x18\x08 \x01(\x0b\x32\x15.swh.graph.OriginDataH\x00\x42\x06\n\x04\x64\x61ta\"N\n\tSuccessor\x12\x12\n\x05swhid\x18\x01 \x01(\tH\x00\x88\x01\x01\x12#\n\x05label\x18\x02 \x03(\x0b\x32\x14.swh.graph.EdgeLabelB\x08\n\x06_swhid\"U\n\x0b\x43ontentData\x12\x13\n\x06length\x18\x01 \x01(\x03H\x00\x88\x01\x01\x12\x17\n\nis_skipped\x18\x02 \x01(\x08H\x01\x88\x01\x01\x42\t\n\x07_lengthB\r\n\x0b_is_skipped\"\xc6\x02\n\x0cRevisionData\x12\x13\n\x06\x61uthor\x18\x01 \x01(\x03H\x00\x88\x01\x01\x12\x18\n\x0b\x61uthor_date\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1f\n\x12\x61uthor_date_offset\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x16\n\tcommitter\x18\x04 \x01(\x03H\x03\x88\x01\x01\x12\x1b\n\x0e\x63ommitter_date\x18\x05 \x01(\x03H\x04\x88\x01\x01\x12\"\n\x15\x63ommitter_date_offset\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x14\n\x07message\x18\x07 \x01(\x0cH\x06\x88\x01\x01\x42\t\n\x07_authorB\x0e\n\x0c_author_dateB\x15\n\x13_author_date_offsetB\x0c\n\n_committerB\x11\n\x0f_committer_dateB\x18\n\x16_committer_date_offsetB\n\n\x08_message\"\xcd\x01\n\x0bReleaseData\x12\x13\n\x06\x61uthor\x18\x01 \x01(\x03H\x00\x88\x01\x01\x12\x18\n\x0b\x61uthor_date\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1f\n\x12\x61uthor_date_offset\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x11\n\x04name\x18\x04 \x01(\x0cH\x03\x88\x01\x01\x12\x14\n\x07message\x18\x05 \x01(\x0cH\x04\x88\x01\x01\x42\t\n\x07_authorB\x0e\n\x0c_author_dateB\x15\n\x13_author_date_offsetB\x07\n\x05_nameB\n\n\x08_message\"&\n\nOriginData\x12\x10\n\x03url\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x06\n\x04_url\"-\n\tEdgeLabel\x12\x0c\n\x04name\x18\x01 \x01(\x0c\x12\x12\n\npermission\x18\x02 \x01(\x05\"\x1e\n\rCountResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\"\x0e\n\x0cStatsRequest\"\x95\x02\n\rStatsResponse\x12\x11\n\tnum_nodes\x18\x01 \x01(\x03\x12\x11\n\tnum_edges\x18\x02 \x01(\x03\x12\x13\n\x0b\x63ompression\x18\x03 \x01(\x01\x12\x15\n\rbits_per_node\x18\x04 \x01(\x01\x12\x15\n\rbits_per_edge\x18\x05 \x01(\x01\x12\x14\n\x0c\x61vg_locality\x18\x06 \x01(\x01\x12\x14\n\x0cindegree_min\x18\x07 \x01(\x03\x12\x14\n\x0cindegree_max\x18\x08 \x01(\x03\x12\x14\n\x0cindegree_avg\x18\t \x01(\x01\x12\x15\n\routdegree_min\x18\n \x01(\x03\x12\x15\n\routdegree_max\x18\x0b \x01(\x03\x12\x15\n\routdegree_avg\x18\x0c \x01(\x01\"\"\n\x11\x43heckSwhidRequest\x12\r\n\x05swhid\x18\x01 \x01(\t\"W\n\x0eGetNodeRequest\x12\r\n\x05swhid\x18\x01 \x01(\t\x12-\n\x04mask\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskH\x00\x88\x01\x01\x42\x07\n\x05_mask\"5\n\x12\x43heckSwhidResponse\x12\x0e\n\x06\x65xists\x18\x01 \x01(\x08\x12\x0f\n\x07\x64\x65tails\x18\x02 \x01(\t*5\n\x0eGraphDirection\x12\x0b\n\x07\x46ORWARD\x10\x00\x12\x0c\n\x08\x42\x41\x43KWARD\x10\x01\x12\x08\n\x04\x42OTH\x10\x02\x32\x96\x03\n\x10TraversalService\x12:\n\x08Traverse\x12\x1b.swh.graph.TraversalRequest\x1a\x0f.swh.graph.Node0\x01\x12\x43\n\nCountNodes\x12\x1b.swh.graph.TraversalRequest\x1a\x18.swh.graph.CountResponse\x12\x43\n\nCountEdges\x12\x1b.swh.graph.TraversalRequest\x1a\x18.swh.graph.CountResponse\x12:\n\x05Stats\x12\x17.swh.graph.StatsRequest\x1a\x18.swh.graph.StatsResponse\x12I\n\nCheckSwhid\x12\x1c.swh.graph.CheckSwhidRequest\x1a\x1d.swh.graph.CheckSwhidResponse\x12\x35\n\x07GetNode\x12\x19.swh.graph.GetNodeRequest\x1a\x0f.swh.graph.NodeB0\n\x1eorg.softwareheritage.graph.rpcB\x0cGraphServiceP\x01\x62\x06proto3') _GRAPHDIRECTION = DESCRIPTOR.enum_types_by_name['GraphDirection'] GraphDirection = enum_type_wrapper.EnumTypeWrapper(_GRAPHDIRECTION) FORWARD = 0 BACKWARD = 1 BOTH = 2 _TRAVERSALREQUEST = DESCRIPTOR.message_types_by_name['TraversalRequest'] _NODEFILTER = DESCRIPTOR.message_types_by_name['NodeFilter'] -_NODEFIELDS = DESCRIPTOR.message_types_by_name['NodeFields'] _NODE = DESCRIPTOR.message_types_by_name['Node'] _SUCCESSOR = DESCRIPTOR.message_types_by_name['Successor'] +_CONTENTDATA = DESCRIPTOR.message_types_by_name['ContentData'] +_REVISIONDATA = DESCRIPTOR.message_types_by_name['RevisionData'] +_RELEASEDATA = DESCRIPTOR.message_types_by_name['ReleaseData'] +_ORIGINDATA = DESCRIPTOR.message_types_by_name['OriginData'] _EDGELABEL = DESCRIPTOR.message_types_by_name['EdgeLabel'] _COUNTRESPONSE = DESCRIPTOR.message_types_by_name['CountResponse'] _STATSREQUEST = DESCRIPTOR.message_types_by_name['StatsRequest'] _STATSRESPONSE = DESCRIPTOR.message_types_by_name['StatsResponse'] _CHECKSWHIDREQUEST = DESCRIPTOR.message_types_by_name['CheckSwhidRequest'] +_GETNODEREQUEST = DESCRIPTOR.message_types_by_name['GetNodeRequest'] _CHECKSWHIDRESPONSE = DESCRIPTOR.message_types_by_name['CheckSwhidResponse'] TraversalRequest = _reflection.GeneratedProtocolMessageType('TraversalRequest', (_message.Message,), { 'DESCRIPTOR' : _TRAVERSALREQUEST, '__module__' : 'swh.graph.rpc.swhgraph_pb2' # @@protoc_insertion_point(class_scope:swh.graph.TraversalRequest) }) _sym_db.RegisterMessage(TraversalRequest) NodeFilter = _reflection.GeneratedProtocolMessageType('NodeFilter', (_message.Message,), { 'DESCRIPTOR' : _NODEFILTER, '__module__' : 'swh.graph.rpc.swhgraph_pb2' # @@protoc_insertion_point(class_scope:swh.graph.NodeFilter) }) _sym_db.RegisterMessage(NodeFilter) -NodeFields = _reflection.GeneratedProtocolMessageType('NodeFields', (_message.Message,), { - 'DESCRIPTOR' : _NODEFIELDS, - '__module__' : 'swh.graph.rpc.swhgraph_pb2' - # @@protoc_insertion_point(class_scope:swh.graph.NodeFields) - }) -_sym_db.RegisterMessage(NodeFields) - Node = _reflection.GeneratedProtocolMessageType('Node', (_message.Message,), { 'DESCRIPTOR' : _NODE, '__module__' : 'swh.graph.rpc.swhgraph_pb2' # @@protoc_insertion_point(class_scope:swh.graph.Node) }) _sym_db.RegisterMessage(Node) Successor = _reflection.GeneratedProtocolMessageType('Successor', (_message.Message,), { 'DESCRIPTOR' : _SUCCESSOR, '__module__' : 'swh.graph.rpc.swhgraph_pb2' # @@protoc_insertion_point(class_scope:swh.graph.Successor) }) _sym_db.RegisterMessage(Successor) +ContentData = _reflection.GeneratedProtocolMessageType('ContentData', (_message.Message,), { + 'DESCRIPTOR' : _CONTENTDATA, + '__module__' : 'swh.graph.rpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.ContentData) + }) +_sym_db.RegisterMessage(ContentData) + +RevisionData = _reflection.GeneratedProtocolMessageType('RevisionData', (_message.Message,), { + 'DESCRIPTOR' : _REVISIONDATA, + '__module__' : 'swh.graph.rpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.RevisionData) + }) +_sym_db.RegisterMessage(RevisionData) + +ReleaseData = _reflection.GeneratedProtocolMessageType('ReleaseData', (_message.Message,), { + 'DESCRIPTOR' : _RELEASEDATA, + '__module__' : 'swh.graph.rpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.ReleaseData) + }) +_sym_db.RegisterMessage(ReleaseData) + +OriginData = _reflection.GeneratedProtocolMessageType('OriginData', (_message.Message,), { + 'DESCRIPTOR' : _ORIGINDATA, + '__module__' : 'swh.graph.rpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.OriginData) + }) +_sym_db.RegisterMessage(OriginData) + EdgeLabel = _reflection.GeneratedProtocolMessageType('EdgeLabel', (_message.Message,), { 'DESCRIPTOR' : _EDGELABEL, '__module__' : 'swh.graph.rpc.swhgraph_pb2' # @@protoc_insertion_point(class_scope:swh.graph.EdgeLabel) }) _sym_db.RegisterMessage(EdgeLabel) CountResponse = _reflection.GeneratedProtocolMessageType('CountResponse', (_message.Message,), { 'DESCRIPTOR' : _COUNTRESPONSE, '__module__' : 'swh.graph.rpc.swhgraph_pb2' # @@protoc_insertion_point(class_scope:swh.graph.CountResponse) }) _sym_db.RegisterMessage(CountResponse) StatsRequest = _reflection.GeneratedProtocolMessageType('StatsRequest', (_message.Message,), { 'DESCRIPTOR' : _STATSREQUEST, '__module__' : 'swh.graph.rpc.swhgraph_pb2' # @@protoc_insertion_point(class_scope:swh.graph.StatsRequest) }) _sym_db.RegisterMessage(StatsRequest) StatsResponse = _reflection.GeneratedProtocolMessageType('StatsResponse', (_message.Message,), { 'DESCRIPTOR' : _STATSRESPONSE, '__module__' : 'swh.graph.rpc.swhgraph_pb2' # @@protoc_insertion_point(class_scope:swh.graph.StatsResponse) }) _sym_db.RegisterMessage(StatsResponse) CheckSwhidRequest = _reflection.GeneratedProtocolMessageType('CheckSwhidRequest', (_message.Message,), { 'DESCRIPTOR' : _CHECKSWHIDREQUEST, '__module__' : 'swh.graph.rpc.swhgraph_pb2' # @@protoc_insertion_point(class_scope:swh.graph.CheckSwhidRequest) }) _sym_db.RegisterMessage(CheckSwhidRequest) +GetNodeRequest = _reflection.GeneratedProtocolMessageType('GetNodeRequest', (_message.Message,), { + 'DESCRIPTOR' : _GETNODEREQUEST, + '__module__' : 'swh.graph.rpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.GetNodeRequest) + }) +_sym_db.RegisterMessage(GetNodeRequest) + CheckSwhidResponse = _reflection.GeneratedProtocolMessageType('CheckSwhidResponse', (_message.Message,), { 'DESCRIPTOR' : _CHECKSWHIDRESPONSE, '__module__' : 'swh.graph.rpc.swhgraph_pb2' # @@protoc_insertion_point(class_scope:swh.graph.CheckSwhidResponse) }) _sym_db.RegisterMessage(CheckSwhidResponse) _TRAVERSALSERVICE = DESCRIPTOR.services_by_name['TraversalService'] if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'\n\036org.softwareheritage.graph.rpcB\014GraphServiceP\001' - _GRAPHDIRECTION._serialized_start=2841 - _GRAPHDIRECTION._serialized_end=2894 - _TRAVERSALREQUEST._serialized_start=44 - _TRAVERSALREQUEST._serialized_end=420 - _NODEFILTER._serialized_start=423 - _NODEFILTER._serialized_end=601 - _NODEFIELDS._serialized_start=604 - _NODEFIELDS._serialized_end=1506 - _NODE._serialized_start=1509 - _NODE._serialized_end=2293 - _SUCCESSOR._serialized_start=2295 - _SUCCESSOR._serialized_end=2373 - _EDGELABEL._serialized_start=2375 - _EDGELABEL._serialized_end=2420 - _COUNTRESPONSE._serialized_start=2422 - _COUNTRESPONSE._serialized_end=2452 - _STATSREQUEST._serialized_start=2454 - _STATSREQUEST._serialized_end=2468 - _STATSRESPONSE._serialized_start=2471 - _STATSRESPONSE._serialized_end=2748 - _CHECKSWHIDREQUEST._serialized_start=2750 - _CHECKSWHIDREQUEST._serialized_end=2784 - _CHECKSWHIDRESPONSE._serialized_start=2786 - _CHECKSWHIDRESPONSE._serialized_end=2839 - _TRAVERSALSERVICE._serialized_start=2897 - _TRAVERSALSERVICE._serialized_end=3248 + _GRAPHDIRECTION._serialized_start=2133 + _GRAPHDIRECTION._serialized_end=2186 + _TRAVERSALREQUEST._serialized_start=78 + _TRAVERSALREQUEST._serialized_end=422 + _NODEFILTER._serialized_start=425 + _NODEFILTER._serialized_end=603 + _NODE._serialized_start=606 + _NODE._serialized_end=832 + _SUCCESSOR._serialized_start=834 + _SUCCESSOR._serialized_end=912 + _CONTENTDATA._serialized_start=914 + _CONTENTDATA._serialized_end=999 + _REVISIONDATA._serialized_start=1002 + _REVISIONDATA._serialized_end=1328 + _RELEASEDATA._serialized_start=1331 + _RELEASEDATA._serialized_end=1536 + _ORIGINDATA._serialized_start=1538 + _ORIGINDATA._serialized_end=1576 + _EDGELABEL._serialized_start=1578 + _EDGELABEL._serialized_end=1623 + _COUNTRESPONSE._serialized_start=1625 + _COUNTRESPONSE._serialized_end=1655 + _STATSREQUEST._serialized_start=1657 + _STATSREQUEST._serialized_end=1671 + _STATSRESPONSE._serialized_start=1674 + _STATSRESPONSE._serialized_end=1951 + _CHECKSWHIDREQUEST._serialized_start=1953 + _CHECKSWHIDREQUEST._serialized_end=1987 + _GETNODEREQUEST._serialized_start=1989 + _GETNODEREQUEST._serialized_end=2076 + _CHECKSWHIDRESPONSE._serialized_start=2078 + _CHECKSWHIDRESPONSE._serialized_end=2131 + _TRAVERSALSERVICE._serialized_start=2189 + _TRAVERSALSERVICE._serialized_end=2595 # @@protoc_insertion_point(module_scope) diff --git a/swh/graph/rpc/swhgraph_pb2.pyi b/swh/graph/rpc/swhgraph_pb2.pyi index c41ba12..01cd94f 100644 --- a/swh/graph/rpc/swhgraph_pb2.pyi +++ b/swh/graph/rpc/swhgraph_pb2.pyi @@ -1,418 +1,382 @@ """ @generated by mypy-protobuf. Do not edit manually! isort:skip_file """ import builtins import google.protobuf.descriptor +import google.protobuf.field_mask_pb2 import google.protobuf.internal.containers import google.protobuf.internal.enum_type_wrapper import google.protobuf.message import typing import typing_extensions DESCRIPTOR: google.protobuf.descriptor.FileDescriptor class _GraphDirection: ValueType = typing.NewType('ValueType', builtins.int) V: typing_extensions.TypeAlias = ValueType class _GraphDirectionEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_GraphDirection.ValueType], builtins.type): DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor FORWARD: _GraphDirection.ValueType # 0 BACKWARD: _GraphDirection.ValueType # 1 BOTH: _GraphDirection.ValueType # 2 class GraphDirection(_GraphDirection, metaclass=_GraphDirectionEnumTypeWrapper): pass FORWARD: GraphDirection.ValueType # 0 BACKWARD: GraphDirection.ValueType # 1 BOTH: GraphDirection.ValueType # 2 global___GraphDirection = GraphDirection class TraversalRequest(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor SRC_FIELD_NUMBER: builtins.int DIRECTION_FIELD_NUMBER: builtins.int EDGES_FIELD_NUMBER: builtins.int MAX_EDGES_FIELD_NUMBER: builtins.int MIN_DEPTH_FIELD_NUMBER: builtins.int MAX_DEPTH_FIELD_NUMBER: builtins.int RETURN_NODES_FIELD_NUMBER: builtins.int - RETURN_FIELDS_FIELD_NUMBER: builtins.int + MASK_FIELD_NUMBER: builtins.int @property def src(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[typing.Text]: ... direction: global___GraphDirection.ValueType """Traversal options""" edges: typing.Text max_edges: builtins.int min_depth: builtins.int max_depth: builtins.int @property def return_nodes(self) -> global___NodeFilter: ... @property - def return_fields(self) -> global___NodeFields: ... + def mask(self) -> google.protobuf.field_mask_pb2.FieldMask: ... def __init__(self, *, src: typing.Optional[typing.Iterable[typing.Text]] = ..., - direction: typing.Optional[global___GraphDirection.ValueType] = ..., + direction: global___GraphDirection.ValueType = ..., edges: typing.Optional[typing.Text] = ..., max_edges: typing.Optional[builtins.int] = ..., min_depth: typing.Optional[builtins.int] = ..., max_depth: typing.Optional[builtins.int] = ..., return_nodes: typing.Optional[global___NodeFilter] = ..., - return_fields: typing.Optional[global___NodeFields] = ..., + mask: typing.Optional[google.protobuf.field_mask_pb2.FieldMask] = ..., ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_direction",b"_direction","_edges",b"_edges","_max_depth",b"_max_depth","_max_edges",b"_max_edges","_min_depth",b"_min_depth","_return_fields",b"_return_fields","_return_nodes",b"_return_nodes","direction",b"direction","edges",b"edges","max_depth",b"max_depth","max_edges",b"max_edges","min_depth",b"min_depth","return_fields",b"return_fields","return_nodes",b"return_nodes"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_direction",b"_direction","_edges",b"_edges","_max_depth",b"_max_depth","_max_edges",b"_max_edges","_min_depth",b"_min_depth","_return_fields",b"_return_fields","_return_nodes",b"_return_nodes","direction",b"direction","edges",b"edges","max_depth",b"max_depth","max_edges",b"max_edges","min_depth",b"min_depth","return_fields",b"return_fields","return_nodes",b"return_nodes","src",b"src"]) -> None: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_direction",b"_direction"]) -> typing.Optional[typing_extensions.Literal["direction"]]: ... + def HasField(self, field_name: typing_extensions.Literal["_edges",b"_edges","_mask",b"_mask","_max_depth",b"_max_depth","_max_edges",b"_max_edges","_min_depth",b"_min_depth","_return_nodes",b"_return_nodes","edges",b"edges","mask",b"mask","max_depth",b"max_depth","max_edges",b"max_edges","min_depth",b"min_depth","return_nodes",b"return_nodes"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_edges",b"_edges","_mask",b"_mask","_max_depth",b"_max_depth","_max_edges",b"_max_edges","_min_depth",b"_min_depth","_return_nodes",b"_return_nodes","direction",b"direction","edges",b"edges","mask",b"mask","max_depth",b"max_depth","max_edges",b"max_edges","min_depth",b"min_depth","return_nodes",b"return_nodes","src",b"src"]) -> None: ... @typing.overload def WhichOneof(self, oneof_group: typing_extensions.Literal["_edges",b"_edges"]) -> typing.Optional[typing_extensions.Literal["edges"]]: ... @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_mask",b"_mask"]) -> typing.Optional[typing_extensions.Literal["mask"]]: ... + @typing.overload def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_depth",b"_max_depth"]) -> typing.Optional[typing_extensions.Literal["max_depth"]]: ... @typing.overload def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_edges",b"_max_edges"]) -> typing.Optional[typing_extensions.Literal["max_edges"]]: ... @typing.overload def WhichOneof(self, oneof_group: typing_extensions.Literal["_min_depth",b"_min_depth"]) -> typing.Optional[typing_extensions.Literal["min_depth"]]: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_return_fields",b"_return_fields"]) -> typing.Optional[typing_extensions.Literal["return_fields"]]: ... - @typing.overload def WhichOneof(self, oneof_group: typing_extensions.Literal["_return_nodes",b"_return_nodes"]) -> typing.Optional[typing_extensions.Literal["return_nodes"]]: ... global___TraversalRequest = TraversalRequest class NodeFilter(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor TYPES_FIELD_NUMBER: builtins.int MIN_TRAVERSAL_SUCCESSORS_FIELD_NUMBER: builtins.int MAX_TRAVERSAL_SUCCESSORS_FIELD_NUMBER: builtins.int types: typing.Text min_traversal_successors: builtins.int max_traversal_successors: builtins.int def __init__(self, *, types: typing.Optional[typing.Text] = ..., min_traversal_successors: typing.Optional[builtins.int] = ..., max_traversal_successors: typing.Optional[builtins.int] = ..., ) -> None: ... def HasField(self, field_name: typing_extensions.Literal["_max_traversal_successors",b"_max_traversal_successors","_min_traversal_successors",b"_min_traversal_successors","_types",b"_types","max_traversal_successors",b"max_traversal_successors","min_traversal_successors",b"min_traversal_successors","types",b"types"]) -> builtins.bool: ... def ClearField(self, field_name: typing_extensions.Literal["_max_traversal_successors",b"_max_traversal_successors","_min_traversal_successors",b"_min_traversal_successors","_types",b"_types","max_traversal_successors",b"max_traversal_successors","min_traversal_successors",b"min_traversal_successors","types",b"types"]) -> None: ... @typing.overload def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_traversal_successors",b"_max_traversal_successors"]) -> typing.Optional[typing_extensions.Literal["max_traversal_successors"]]: ... @typing.overload def WhichOneof(self, oneof_group: typing_extensions.Literal["_min_traversal_successors",b"_min_traversal_successors"]) -> typing.Optional[typing_extensions.Literal["min_traversal_successors"]]: ... @typing.overload def WhichOneof(self, oneof_group: typing_extensions.Literal["_types",b"_types"]) -> typing.Optional[typing_extensions.Literal["types"]]: ... global___NodeFilter = NodeFilter -class NodeFields(google.protobuf.message.Message): +class Node(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor SWHID_FIELD_NUMBER: builtins.int SUCCESSOR_FIELD_NUMBER: builtins.int - SUCCESSOR_SWHID_FIELD_NUMBER: builtins.int - SUCCESSOR_LABEL_FIELD_NUMBER: builtins.int - CNT_LENGTH_FIELD_NUMBER: builtins.int - CNT_IS_SKIPPED_FIELD_NUMBER: builtins.int - REV_AUTHOR_FIELD_NUMBER: builtins.int - REV_AUTHOR_DATE_FIELD_NUMBER: builtins.int - REV_AUTHOR_DATE_OFFSET_FIELD_NUMBER: builtins.int - REV_COMMITTER_FIELD_NUMBER: builtins.int - REV_COMMITTER_DATE_FIELD_NUMBER: builtins.int - REV_COMMITTER_DATE_OFFSET_FIELD_NUMBER: builtins.int - REV_MESSAGE_FIELD_NUMBER: builtins.int - REL_AUTHOR_FIELD_NUMBER: builtins.int - REL_AUTHOR_DATE_FIELD_NUMBER: builtins.int - REL_AUTHOR_DATE_OFFSET_FIELD_NUMBER: builtins.int - REL_NAME_FIELD_NUMBER: builtins.int - REL_MESSAGE_FIELD_NUMBER: builtins.int - ORI_URL_FIELD_NUMBER: builtins.int - swhid: builtins.bool - successor: builtins.bool - successor_swhid: builtins.bool - successor_label: builtins.bool - cnt_length: builtins.bool - cnt_is_skipped: builtins.bool - rev_author: builtins.bool - rev_author_date: builtins.bool - rev_author_date_offset: builtins.bool - rev_committer: builtins.bool - rev_committer_date: builtins.bool - rev_committer_date_offset: builtins.bool - rev_message: builtins.bool - rel_author: builtins.bool - rel_author_date: builtins.bool - rel_author_date_offset: builtins.bool - rel_name: builtins.bool - rel_message: builtins.bool - ori_url: builtins.bool + CNT_FIELD_NUMBER: builtins.int + REV_FIELD_NUMBER: builtins.int + REL_FIELD_NUMBER: builtins.int + ORI_FIELD_NUMBER: builtins.int + swhid: typing.Text + @property + def successor(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Successor]: ... + @property + def cnt(self) -> global___ContentData: ... + @property + def rev(self) -> global___RevisionData: ... + @property + def rel(self) -> global___ReleaseData: ... + @property + def ori(self) -> global___OriginData: ... def __init__(self, *, - swhid: typing.Optional[builtins.bool] = ..., - successor: typing.Optional[builtins.bool] = ..., - successor_swhid: typing.Optional[builtins.bool] = ..., - successor_label: typing.Optional[builtins.bool] = ..., - cnt_length: typing.Optional[builtins.bool] = ..., - cnt_is_skipped: typing.Optional[builtins.bool] = ..., - rev_author: typing.Optional[builtins.bool] = ..., - rev_author_date: typing.Optional[builtins.bool] = ..., - rev_author_date_offset: typing.Optional[builtins.bool] = ..., - rev_committer: typing.Optional[builtins.bool] = ..., - rev_committer_date: typing.Optional[builtins.bool] = ..., - rev_committer_date_offset: typing.Optional[builtins.bool] = ..., - rev_message: typing.Optional[builtins.bool] = ..., - rel_author: typing.Optional[builtins.bool] = ..., - rel_author_date: typing.Optional[builtins.bool] = ..., - rel_author_date_offset: typing.Optional[builtins.bool] = ..., - rel_name: typing.Optional[builtins.bool] = ..., - rel_message: typing.Optional[builtins.bool] = ..., - ori_url: typing.Optional[builtins.bool] = ..., + swhid: typing.Text = ..., + successor: typing.Optional[typing.Iterable[global___Successor]] = ..., + cnt: typing.Optional[global___ContentData] = ..., + rev: typing.Optional[global___RevisionData] = ..., + rel: typing.Optional[global___ReleaseData] = ..., + ori: typing.Optional[global___OriginData] = ..., ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_cnt_is_skipped",b"_cnt_is_skipped","_cnt_length",b"_cnt_length","_ori_url",b"_ori_url","_rel_author",b"_rel_author","_rel_author_date",b"_rel_author_date","_rel_author_date_offset",b"_rel_author_date_offset","_rel_message",b"_rel_message","_rel_name",b"_rel_name","_rev_author",b"_rev_author","_rev_author_date",b"_rev_author_date","_rev_author_date_offset",b"_rev_author_date_offset","_rev_committer",b"_rev_committer","_rev_committer_date",b"_rev_committer_date","_rev_committer_date_offset",b"_rev_committer_date_offset","_rev_message",b"_rev_message","_successor",b"_successor","_successor_label",b"_successor_label","_successor_swhid",b"_successor_swhid","_swhid",b"_swhid","cnt_is_skipped",b"cnt_is_skipped","cnt_length",b"cnt_length","ori_url",b"ori_url","rel_author",b"rel_author","rel_author_date",b"rel_author_date","rel_author_date_offset",b"rel_author_date_offset","rel_message",b"rel_message","rel_name",b"rel_name","rev_author",b"rev_author","rev_author_date",b"rev_author_date","rev_author_date_offset",b"rev_author_date_offset","rev_committer",b"rev_committer","rev_committer_date",b"rev_committer_date","rev_committer_date_offset",b"rev_committer_date_offset","rev_message",b"rev_message","successor",b"successor","successor_label",b"successor_label","successor_swhid",b"successor_swhid","swhid",b"swhid"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_cnt_is_skipped",b"_cnt_is_skipped","_cnt_length",b"_cnt_length","_ori_url",b"_ori_url","_rel_author",b"_rel_author","_rel_author_date",b"_rel_author_date","_rel_author_date_offset",b"_rel_author_date_offset","_rel_message",b"_rel_message","_rel_name",b"_rel_name","_rev_author",b"_rev_author","_rev_author_date",b"_rev_author_date","_rev_author_date_offset",b"_rev_author_date_offset","_rev_committer",b"_rev_committer","_rev_committer_date",b"_rev_committer_date","_rev_committer_date_offset",b"_rev_committer_date_offset","_rev_message",b"_rev_message","_successor",b"_successor","_successor_label",b"_successor_label","_successor_swhid",b"_successor_swhid","_swhid",b"_swhid","cnt_is_skipped",b"cnt_is_skipped","cnt_length",b"cnt_length","ori_url",b"ori_url","rel_author",b"rel_author","rel_author_date",b"rel_author_date","rel_author_date_offset",b"rel_author_date_offset","rel_message",b"rel_message","rel_name",b"rel_name","rev_author",b"rev_author","rev_author_date",b"rev_author_date","rev_author_date_offset",b"rev_author_date_offset","rev_committer",b"rev_committer","rev_committer_date",b"rev_committer_date","rev_committer_date_offset",b"rev_committer_date_offset","rev_message",b"rev_message","successor",b"successor","successor_label",b"successor_label","successor_swhid",b"successor_swhid","swhid",b"swhid"]) -> None: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_cnt_is_skipped",b"_cnt_is_skipped"]) -> typing.Optional[typing_extensions.Literal["cnt_is_skipped"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_cnt_length",b"_cnt_length"]) -> typing.Optional[typing_extensions.Literal["cnt_length"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_ori_url",b"_ori_url"]) -> typing.Optional[typing_extensions.Literal["ori_url"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_author",b"_rel_author"]) -> typing.Optional[typing_extensions.Literal["rel_author"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_author_date",b"_rel_author_date"]) -> typing.Optional[typing_extensions.Literal["rel_author_date"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_author_date_offset",b"_rel_author_date_offset"]) -> typing.Optional[typing_extensions.Literal["rel_author_date_offset"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_message",b"_rel_message"]) -> typing.Optional[typing_extensions.Literal["rel_message"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_name",b"_rel_name"]) -> typing.Optional[typing_extensions.Literal["rel_name"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_author",b"_rev_author"]) -> typing.Optional[typing_extensions.Literal["rev_author"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_author_date",b"_rev_author_date"]) -> typing.Optional[typing_extensions.Literal["rev_author_date"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_author_date_offset",b"_rev_author_date_offset"]) -> typing.Optional[typing_extensions.Literal["rev_author_date_offset"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_committer",b"_rev_committer"]) -> typing.Optional[typing_extensions.Literal["rev_committer"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_committer_date",b"_rev_committer_date"]) -> typing.Optional[typing_extensions.Literal["rev_committer_date"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_committer_date_offset",b"_rev_committer_date_offset"]) -> typing.Optional[typing_extensions.Literal["rev_committer_date_offset"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_message",b"_rev_message"]) -> typing.Optional[typing_extensions.Literal["rev_message"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_successor",b"_successor"]) -> typing.Optional[typing_extensions.Literal["successor"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_successor_label",b"_successor_label"]) -> typing.Optional[typing_extensions.Literal["successor_label"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_successor_swhid",b"_successor_swhid"]) -> typing.Optional[typing_extensions.Literal["successor_swhid"]]: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_swhid",b"_swhid"]) -> typing.Optional[typing_extensions.Literal["swhid"]]: ... -global___NodeFields = NodeFields + def HasField(self, field_name: typing_extensions.Literal["cnt",b"cnt","data",b"data","ori",b"ori","rel",b"rel","rev",b"rev"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["cnt",b"cnt","data",b"data","ori",b"ori","rel",b"rel","rev",b"rev","successor",b"successor","swhid",b"swhid"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["data",b"data"]) -> typing.Optional[typing_extensions.Literal["cnt","rev","rel","ori"]]: ... +global___Node = Node -class Node(google.protobuf.message.Message): +class Successor(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor SWHID_FIELD_NUMBER: builtins.int - SUCCESSOR_FIELD_NUMBER: builtins.int - CNT_LENGTH_FIELD_NUMBER: builtins.int - CNT_IS_SKIPPED_FIELD_NUMBER: builtins.int - REV_AUTHOR_FIELD_NUMBER: builtins.int - REV_AUTHOR_DATE_FIELD_NUMBER: builtins.int - REV_AUTHOR_DATE_OFFSET_FIELD_NUMBER: builtins.int - REV_COMMITTER_FIELD_NUMBER: builtins.int - REV_COMMITTER_DATE_FIELD_NUMBER: builtins.int - REV_COMMITTER_DATE_OFFSET_FIELD_NUMBER: builtins.int - REV_MESSAGE_FIELD_NUMBER: builtins.int - REL_AUTHOR_FIELD_NUMBER: builtins.int - REL_AUTHOR_DATE_FIELD_NUMBER: builtins.int - REL_AUTHOR_DATE_OFFSET_FIELD_NUMBER: builtins.int - REL_NAME_FIELD_NUMBER: builtins.int - REL_MESSAGE_FIELD_NUMBER: builtins.int - ORI_URL_FIELD_NUMBER: builtins.int + LABEL_FIELD_NUMBER: builtins.int swhid: typing.Text @property - def successor(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Successor]: ... - cnt_length: builtins.int - cnt_is_skipped: builtins.bool - rev_author: builtins.int - rev_author_date: builtins.int - rev_author_date_offset: builtins.int - rev_committer: builtins.int - rev_committer_date: builtins.int - rev_committer_date_offset: builtins.int - rev_message: builtins.bytes - rel_author: builtins.int - rel_author_date: builtins.int - rel_author_date_offset: builtins.int - rel_name: builtins.bytes - rel_message: builtins.bytes - ori_url: typing.Text + def label(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___EdgeLabel]: ... def __init__(self, *, - swhid: typing.Text = ..., - successor: typing.Optional[typing.Iterable[global___Successor]] = ..., - cnt_length: typing.Optional[builtins.int] = ..., - cnt_is_skipped: typing.Optional[builtins.bool] = ..., - rev_author: typing.Optional[builtins.int] = ..., - rev_author_date: typing.Optional[builtins.int] = ..., - rev_author_date_offset: typing.Optional[builtins.int] = ..., - rev_committer: typing.Optional[builtins.int] = ..., - rev_committer_date: typing.Optional[builtins.int] = ..., - rev_committer_date_offset: typing.Optional[builtins.int] = ..., - rev_message: typing.Optional[builtins.bytes] = ..., - rel_author: typing.Optional[builtins.int] = ..., - rel_author_date: typing.Optional[builtins.int] = ..., - rel_author_date_offset: typing.Optional[builtins.int] = ..., - rel_name: typing.Optional[builtins.bytes] = ..., - rel_message: typing.Optional[builtins.bytes] = ..., - ori_url: typing.Optional[typing.Text] = ..., + swhid: typing.Optional[typing.Text] = ..., + label: typing.Optional[typing.Iterable[global___EdgeLabel]] = ..., ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_cnt_is_skipped",b"_cnt_is_skipped","_cnt_length",b"_cnt_length","_ori_url",b"_ori_url","_rel_author",b"_rel_author","_rel_author_date",b"_rel_author_date","_rel_author_date_offset",b"_rel_author_date_offset","_rel_message",b"_rel_message","_rel_name",b"_rel_name","_rev_author",b"_rev_author","_rev_author_date",b"_rev_author_date","_rev_author_date_offset",b"_rev_author_date_offset","_rev_committer",b"_rev_committer","_rev_committer_date",b"_rev_committer_date","_rev_committer_date_offset",b"_rev_committer_date_offset","_rev_message",b"_rev_message","cnt_is_skipped",b"cnt_is_skipped","cnt_length",b"cnt_length","ori_url",b"ori_url","rel_author",b"rel_author","rel_author_date",b"rel_author_date","rel_author_date_offset",b"rel_author_date_offset","rel_message",b"rel_message","rel_name",b"rel_name","rev_author",b"rev_author","rev_author_date",b"rev_author_date","rev_author_date_offset",b"rev_author_date_offset","rev_committer",b"rev_committer","rev_committer_date",b"rev_committer_date","rev_committer_date_offset",b"rev_committer_date_offset","rev_message",b"rev_message"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_cnt_is_skipped",b"_cnt_is_skipped","_cnt_length",b"_cnt_length","_ori_url",b"_ori_url","_rel_author",b"_rel_author","_rel_author_date",b"_rel_author_date","_rel_author_date_offset",b"_rel_author_date_offset","_rel_message",b"_rel_message","_rel_name",b"_rel_name","_rev_author",b"_rev_author","_rev_author_date",b"_rev_author_date","_rev_author_date_offset",b"_rev_author_date_offset","_rev_committer",b"_rev_committer","_rev_committer_date",b"_rev_committer_date","_rev_committer_date_offset",b"_rev_committer_date_offset","_rev_message",b"_rev_message","cnt_is_skipped",b"cnt_is_skipped","cnt_length",b"cnt_length","ori_url",b"ori_url","rel_author",b"rel_author","rel_author_date",b"rel_author_date","rel_author_date_offset",b"rel_author_date_offset","rel_message",b"rel_message","rel_name",b"rel_name","rev_author",b"rev_author","rev_author_date",b"rev_author_date","rev_author_date_offset",b"rev_author_date_offset","rev_committer",b"rev_committer","rev_committer_date",b"rev_committer_date","rev_committer_date_offset",b"rev_committer_date_offset","rev_message",b"rev_message","successor",b"successor","swhid",b"swhid"]) -> None: ... - @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_cnt_is_skipped",b"_cnt_is_skipped"]) -> typing.Optional[typing_extensions.Literal["cnt_is_skipped"]]: ... + def HasField(self, field_name: typing_extensions.Literal["_swhid",b"_swhid","swhid",b"swhid"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_swhid",b"_swhid","label",b"label","swhid",b"swhid"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_swhid",b"_swhid"]) -> typing.Optional[typing_extensions.Literal["swhid"]]: ... +global___Successor = Successor + +class ContentData(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + LENGTH_FIELD_NUMBER: builtins.int + IS_SKIPPED_FIELD_NUMBER: builtins.int + length: builtins.int + is_skipped: builtins.bool + def __init__(self, + *, + length: typing.Optional[builtins.int] = ..., + is_skipped: typing.Optional[builtins.bool] = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_is_skipped",b"_is_skipped","_length",b"_length","is_skipped",b"is_skipped","length",b"length"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_is_skipped",b"_is_skipped","_length",b"_length","is_skipped",b"is_skipped","length",b"length"]) -> None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_cnt_length",b"_cnt_length"]) -> typing.Optional[typing_extensions.Literal["cnt_length"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_is_skipped",b"_is_skipped"]) -> typing.Optional[typing_extensions.Literal["is_skipped"]]: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_ori_url",b"_ori_url"]) -> typing.Optional[typing_extensions.Literal["ori_url"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_length",b"_length"]) -> typing.Optional[typing_extensions.Literal["length"]]: ... +global___ContentData = ContentData + +class RevisionData(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + AUTHOR_FIELD_NUMBER: builtins.int + AUTHOR_DATE_FIELD_NUMBER: builtins.int + AUTHOR_DATE_OFFSET_FIELD_NUMBER: builtins.int + COMMITTER_FIELD_NUMBER: builtins.int + COMMITTER_DATE_FIELD_NUMBER: builtins.int + COMMITTER_DATE_OFFSET_FIELD_NUMBER: builtins.int + MESSAGE_FIELD_NUMBER: builtins.int + author: builtins.int + author_date: builtins.int + author_date_offset: builtins.int + committer: builtins.int + committer_date: builtins.int + committer_date_offset: builtins.int + message: builtins.bytes + def __init__(self, + *, + author: typing.Optional[builtins.int] = ..., + author_date: typing.Optional[builtins.int] = ..., + author_date_offset: typing.Optional[builtins.int] = ..., + committer: typing.Optional[builtins.int] = ..., + committer_date: typing.Optional[builtins.int] = ..., + committer_date_offset: typing.Optional[builtins.int] = ..., + message: typing.Optional[builtins.bytes] = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_author",b"_author","_author_date",b"_author_date","_author_date_offset",b"_author_date_offset","_committer",b"_committer","_committer_date",b"_committer_date","_committer_date_offset",b"_committer_date_offset","_message",b"_message","author",b"author","author_date",b"author_date","author_date_offset",b"author_date_offset","committer",b"committer","committer_date",b"committer_date","committer_date_offset",b"committer_date_offset","message",b"message"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_author",b"_author","_author_date",b"_author_date","_author_date_offset",b"_author_date_offset","_committer",b"_committer","_committer_date",b"_committer_date","_committer_date_offset",b"_committer_date_offset","_message",b"_message","author",b"author","author_date",b"author_date","author_date_offset",b"author_date_offset","committer",b"committer","committer_date",b"committer_date","committer_date_offset",b"committer_date_offset","message",b"message"]) -> None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_author",b"_rel_author"]) -> typing.Optional[typing_extensions.Literal["rel_author"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_author",b"_author"]) -> typing.Optional[typing_extensions.Literal["author"]]: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_author_date",b"_rel_author_date"]) -> typing.Optional[typing_extensions.Literal["rel_author_date"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_author_date",b"_author_date"]) -> typing.Optional[typing_extensions.Literal["author_date"]]: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_author_date_offset",b"_rel_author_date_offset"]) -> typing.Optional[typing_extensions.Literal["rel_author_date_offset"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_author_date_offset",b"_author_date_offset"]) -> typing.Optional[typing_extensions.Literal["author_date_offset"]]: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_message",b"_rel_message"]) -> typing.Optional[typing_extensions.Literal["rel_message"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_committer",b"_committer"]) -> typing.Optional[typing_extensions.Literal["committer"]]: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_name",b"_rel_name"]) -> typing.Optional[typing_extensions.Literal["rel_name"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_committer_date",b"_committer_date"]) -> typing.Optional[typing_extensions.Literal["committer_date"]]: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_author",b"_rev_author"]) -> typing.Optional[typing_extensions.Literal["rev_author"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_committer_date_offset",b"_committer_date_offset"]) -> typing.Optional[typing_extensions.Literal["committer_date_offset"]]: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_author_date",b"_rev_author_date"]) -> typing.Optional[typing_extensions.Literal["rev_author_date"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_message",b"_message"]) -> typing.Optional[typing_extensions.Literal["message"]]: ... +global___RevisionData = RevisionData + +class ReleaseData(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + AUTHOR_FIELD_NUMBER: builtins.int + AUTHOR_DATE_FIELD_NUMBER: builtins.int + AUTHOR_DATE_OFFSET_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + MESSAGE_FIELD_NUMBER: builtins.int + author: builtins.int + author_date: builtins.int + author_date_offset: builtins.int + name: builtins.bytes + message: builtins.bytes + def __init__(self, + *, + author: typing.Optional[builtins.int] = ..., + author_date: typing.Optional[builtins.int] = ..., + author_date_offset: typing.Optional[builtins.int] = ..., + name: typing.Optional[builtins.bytes] = ..., + message: typing.Optional[builtins.bytes] = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_author",b"_author","_author_date",b"_author_date","_author_date_offset",b"_author_date_offset","_message",b"_message","_name",b"_name","author",b"author","author_date",b"author_date","author_date_offset",b"author_date_offset","message",b"message","name",b"name"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_author",b"_author","_author_date",b"_author_date","_author_date_offset",b"_author_date_offset","_message",b"_message","_name",b"_name","author",b"author","author_date",b"author_date","author_date_offset",b"author_date_offset","message",b"message","name",b"name"]) -> None: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_author_date_offset",b"_rev_author_date_offset"]) -> typing.Optional[typing_extensions.Literal["rev_author_date_offset"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_author",b"_author"]) -> typing.Optional[typing_extensions.Literal["author"]]: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_committer",b"_rev_committer"]) -> typing.Optional[typing_extensions.Literal["rev_committer"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_author_date",b"_author_date"]) -> typing.Optional[typing_extensions.Literal["author_date"]]: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_committer_date",b"_rev_committer_date"]) -> typing.Optional[typing_extensions.Literal["rev_committer_date"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_author_date_offset",b"_author_date_offset"]) -> typing.Optional[typing_extensions.Literal["author_date_offset"]]: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_committer_date_offset",b"_rev_committer_date_offset"]) -> typing.Optional[typing_extensions.Literal["rev_committer_date_offset"]]: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_message",b"_message"]) -> typing.Optional[typing_extensions.Literal["message"]]: ... @typing.overload - def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_message",b"_rev_message"]) -> typing.Optional[typing_extensions.Literal["rev_message"]]: ... -global___Node = Node + def WhichOneof(self, oneof_group: typing_extensions.Literal["_name",b"_name"]) -> typing.Optional[typing_extensions.Literal["name"]]: ... +global___ReleaseData = ReleaseData -class Successor(google.protobuf.message.Message): +class OriginData(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor - SWHID_FIELD_NUMBER: builtins.int - LABEL_FIELD_NUMBER: builtins.int - swhid: typing.Text - @property - def label(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___EdgeLabel]: ... + URL_FIELD_NUMBER: builtins.int + url: typing.Text def __init__(self, *, - swhid: typing.Optional[typing.Text] = ..., - label: typing.Optional[typing.Iterable[global___EdgeLabel]] = ..., + url: typing.Optional[typing.Text] = ..., ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_swhid",b"_swhid","swhid",b"swhid"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_swhid",b"_swhid","label",b"label","swhid",b"swhid"]) -> None: ... - def WhichOneof(self, oneof_group: typing_extensions.Literal["_swhid",b"_swhid"]) -> typing.Optional[typing_extensions.Literal["swhid"]]: ... -global___Successor = Successor + def HasField(self, field_name: typing_extensions.Literal["_url",b"_url","url",b"url"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_url",b"_url","url",b"url"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_url",b"_url"]) -> typing.Optional[typing_extensions.Literal["url"]]: ... +global___OriginData = OriginData class EdgeLabel(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor NAME_FIELD_NUMBER: builtins.int PERMISSION_FIELD_NUMBER: builtins.int name: builtins.bytes permission: builtins.int def __init__(self, *, name: builtins.bytes = ..., permission: builtins.int = ..., ) -> None: ... def ClearField(self, field_name: typing_extensions.Literal["name",b"name","permission",b"permission"]) -> None: ... global___EdgeLabel = EdgeLabel class CountResponse(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor COUNT_FIELD_NUMBER: builtins.int count: builtins.int def __init__(self, *, count: builtins.int = ..., ) -> None: ... def ClearField(self, field_name: typing_extensions.Literal["count",b"count"]) -> None: ... global___CountResponse = CountResponse class StatsRequest(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor def __init__(self, ) -> None: ... global___StatsRequest = StatsRequest class StatsResponse(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor NUM_NODES_FIELD_NUMBER: builtins.int NUM_EDGES_FIELD_NUMBER: builtins.int COMPRESSION_FIELD_NUMBER: builtins.int BITS_PER_NODE_FIELD_NUMBER: builtins.int BITS_PER_EDGE_FIELD_NUMBER: builtins.int AVG_LOCALITY_FIELD_NUMBER: builtins.int INDEGREE_MIN_FIELD_NUMBER: builtins.int INDEGREE_MAX_FIELD_NUMBER: builtins.int INDEGREE_AVG_FIELD_NUMBER: builtins.int OUTDEGREE_MIN_FIELD_NUMBER: builtins.int OUTDEGREE_MAX_FIELD_NUMBER: builtins.int OUTDEGREE_AVG_FIELD_NUMBER: builtins.int num_nodes: builtins.int num_edges: builtins.int compression: builtins.float bits_per_node: builtins.float bits_per_edge: builtins.float avg_locality: builtins.float indegree_min: builtins.int indegree_max: builtins.int indegree_avg: builtins.float outdegree_min: builtins.int outdegree_max: builtins.int outdegree_avg: builtins.float def __init__(self, *, num_nodes: builtins.int = ..., num_edges: builtins.int = ..., compression: builtins.float = ..., bits_per_node: builtins.float = ..., bits_per_edge: builtins.float = ..., avg_locality: builtins.float = ..., indegree_min: builtins.int = ..., indegree_max: builtins.int = ..., indegree_avg: builtins.float = ..., outdegree_min: builtins.int = ..., outdegree_max: builtins.int = ..., outdegree_avg: builtins.float = ..., ) -> None: ... def ClearField(self, field_name: typing_extensions.Literal["avg_locality",b"avg_locality","bits_per_edge",b"bits_per_edge","bits_per_node",b"bits_per_node","compression",b"compression","indegree_avg",b"indegree_avg","indegree_max",b"indegree_max","indegree_min",b"indegree_min","num_edges",b"num_edges","num_nodes",b"num_nodes","outdegree_avg",b"outdegree_avg","outdegree_max",b"outdegree_max","outdegree_min",b"outdegree_min"]) -> None: ... global___StatsResponse = StatsResponse class CheckSwhidRequest(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor SWHID_FIELD_NUMBER: builtins.int swhid: typing.Text def __init__(self, *, swhid: typing.Text = ..., ) -> None: ... def ClearField(self, field_name: typing_extensions.Literal["swhid",b"swhid"]) -> None: ... global___CheckSwhidRequest = CheckSwhidRequest +class GetNodeRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + SWHID_FIELD_NUMBER: builtins.int + MASK_FIELD_NUMBER: builtins.int + swhid: typing.Text + @property + def mask(self) -> google.protobuf.field_mask_pb2.FieldMask: ... + def __init__(self, + *, + swhid: typing.Text = ..., + mask: typing.Optional[google.protobuf.field_mask_pb2.FieldMask] = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_mask",b"_mask","mask",b"mask"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_mask",b"_mask","mask",b"mask","swhid",b"swhid"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_mask",b"_mask"]) -> typing.Optional[typing_extensions.Literal["mask"]]: ... +global___GetNodeRequest = GetNodeRequest + class CheckSwhidResponse(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor EXISTS_FIELD_NUMBER: builtins.int DETAILS_FIELD_NUMBER: builtins.int exists: builtins.bool details: typing.Text def __init__(self, *, exists: builtins.bool = ..., details: typing.Text = ..., ) -> None: ... def ClearField(self, field_name: typing_extensions.Literal["details",b"details","exists",b"exists"]) -> None: ... global___CheckSwhidResponse = CheckSwhidResponse diff --git a/swh/graph/rpc/swhgraph_pb2_grpc.py b/swh/graph/rpc/swhgraph_pb2_grpc.py index b906c52..9602609 100644 --- a/swh/graph/rpc/swhgraph_pb2_grpc.py +++ b/swh/graph/rpc/swhgraph_pb2_grpc.py @@ -1,198 +1,231 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc from swh.graph.rpc import swhgraph_pb2 as swh_dot_graph_dot_rpc_dot_swhgraph__pb2 class TraversalServiceStub(object): """Missing associated documentation comment in .proto file.""" def __init__(self, channel): """Constructor. Args: channel: A grpc.Channel. """ self.Traverse = channel.unary_stream( '/swh.graph.TraversalService/Traverse', request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.FromString, ) self.CountNodes = channel.unary_unary( '/swh.graph.TraversalService/CountNodes', request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.FromString, ) self.CountEdges = channel.unary_unary( '/swh.graph.TraversalService/CountEdges', request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.FromString, ) self.Stats = channel.unary_unary( '/swh.graph.TraversalService/Stats', request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsRequest.SerializeToString, response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsResponse.FromString, ) self.CheckSwhid = channel.unary_unary( '/swh.graph.TraversalService/CheckSwhid', request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CheckSwhidRequest.SerializeToString, response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CheckSwhidResponse.FromString, ) + self.GetNode = channel.unary_unary( + '/swh.graph.TraversalService/GetNode', + request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.GetNodeRequest.SerializeToString, + response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.FromString, + ) class TraversalServiceServicer(object): """Missing associated documentation comment in .proto file.""" def Traverse(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def CountNodes(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def CountEdges(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def Stats(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def CheckSwhid(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def GetNode(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def add_TraversalServiceServicer_to_server(servicer, server): rpc_method_handlers = { 'Traverse': grpc.unary_stream_rpc_method_handler( servicer.Traverse, request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.FromString, response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.SerializeToString, ), 'CountNodes': grpc.unary_unary_rpc_method_handler( servicer.CountNodes, request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.FromString, response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.SerializeToString, ), 'CountEdges': grpc.unary_unary_rpc_method_handler( servicer.CountEdges, request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.FromString, response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.SerializeToString, ), 'Stats': grpc.unary_unary_rpc_method_handler( servicer.Stats, request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsRequest.FromString, response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsResponse.SerializeToString, ), 'CheckSwhid': grpc.unary_unary_rpc_method_handler( servicer.CheckSwhid, request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CheckSwhidRequest.FromString, response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CheckSwhidResponse.SerializeToString, ), + 'GetNode': grpc.unary_unary_rpc_method_handler( + servicer.GetNode, + request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.GetNodeRequest.FromString, + response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'swh.graph.TraversalService', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) # This class is part of an EXPERIMENTAL API. class TraversalService(object): """Missing associated documentation comment in .proto file.""" @staticmethod def Traverse(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_stream(request, target, '/swh.graph.TraversalService/Traverse', swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod def CountNodes(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/CountNodes', swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod def CountEdges(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/CountEdges', swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod def Stats(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/Stats', swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsRequest.SerializeToString, swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod def CheckSwhid(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/CheckSwhid', swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CheckSwhidRequest.SerializeToString, swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CheckSwhidResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetNode(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/GetNode', + swh_dot_graph_dot_rpc_dot_swhgraph__pb2.GetNodeRequest.SerializeToString, + swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata)