diff --git a/java/pom.xml b/java/pom.xml
index 9322d6a..234b3b9 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -1,361 +1,420 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.softwareheritage.graph</groupId>
<artifactId>swh-graph</artifactId>
<version>${git.closest.tag.name}</version>
<name>swh-graph</name>
<url>https://forge.softwareheritage.org/source/swh-graph/</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.release>11</maven.compiler.release>
+ <protobuf.version>3.20.1</protobuf.version>
+ <grpc.version>1.46.0</grpc.version>
</properties>
<dependencies>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>5.7.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<version>5.7.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>5.7.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest</artifactId>
<version>2.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.javalin</groupId>
<artifactId>javalin</artifactId>
<version>3.0.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>1.7.26</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.13.0</version>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>webgraph-big</artifactId>
<version>3.6.7</version>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
<version>8.5.8</version>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>dsiutils</artifactId>
<version>2.7.1</version>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>sux4j</artifactId>
<version>5.3.1</version>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>law</artifactId>
<version>2.7.2</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.umlgraph</groupId>
<artifactId>umlgraph</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty.aggregate</groupId>
<artifactId>jetty-all</artifactId>
</exclusion>
<exclusion>
<groupId>it.unimi.di</groupId>
<artifactId>mg4j</artifactId>
</exclusion>
<exclusion>
<groupId>it.unimi.di</groupId>
<artifactId>mg4j-big</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.martiansoftware</groupId>
<artifactId>jsap</artifactId>
<version>2.1</version>
</dependency>
<dependency>
<groupId>net.sf.py4j</groupId>
<artifactId>py4j</artifactId>
<version>0.10.9.3</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.15</version>
</dependency>
<dependency>
<groupId>com.github.luben</groupId>
<artifactId>zstd-jni</artifactId>
<version>1.5.1-1</version>
</dependency>
<dependency>
<groupId>org.apache.orc</groupId>
<artifactId>orc-core</artifactId>
<version>1.7.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-runtime</artifactId>
<version>3.3.1</version>
</dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>${protobuf.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-netty-shaded</artifactId>
+ <version>${grpc.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf</artifactId>
+ <version>${grpc.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-stub</artifactId>
+ <version>${grpc.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-services</artifactId>
+ <version>${grpc.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.annotation</groupId>
+ <artifactId>javax.annotation-api</artifactId>
+ <version>1.3.2</version>
+ </dependency>
</dependencies>
<build>
<pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
<plugins>
<!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>3.1.0</version>
</plugin>
<!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<source>11</source>
<target>11</target>
<compilerArgs>
<arg>-verbose</arg>
<arg>-Xlint:all</arg>
</compilerArgs>
</configuration>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.2</version>
</plugin>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.22.2</version>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
<!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
<plugin>
<artifactId>maven-site-plugin</artifactId>
<version>3.7.1</version>
</plugin>
<plugin>
<artifactId>maven-project-info-reports-plugin</artifactId>
<version>3.0.0</version>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.3.0</version>
<configuration>
<archive>
<manifest>
<mainClass>org.softwareheritage.graph.server.App</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<appendAssemblyId>false</appendAssemblyId>
</configuration>
<executions>
<execution>
<id>make-assembly</id> <!-- this is used for inheritance merges -->
<phase>package</phase> <!-- bind to the packaging phase -->
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Spotless code formatting tool -->
<plugin>
<groupId>com.diffplug.spotless</groupId>
<artifactId>spotless-maven-plugin</artifactId>
<version>2.22.1</version>
<configuration>
<formats>
<format>
<includes>
<include>*.md</include>
<include>.gitignore</include>
</includes>
<trimTrailingWhitespace/>
<endWithNewline/>
<indent>
<spaces>true</spaces>
<spacesPerTab>4</spacesPerTab>
</indent>
</format>
</formats>
<java>
<removeUnusedImports/>
<eclipse>
<version>4.16.0</version>
<file>.coding-style.xml</file>
</eclipse>
</java>
</configuration>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<version>3.0.1</version>
<executions>
<execution>
<id>get-the-git-infos</id>
<goals>
<goal>revision</goal>
</goals>
<phase>initialize</phase>
</execution>
</executions>
<configuration>
<verbose>true</verbose>
<offline>true</offline>
<useNativeGit>true</useNativeGit>
<gitDescribe>
<always>true</always>
<match>v*</match>
</gitDescribe>
<replacementProperties>
<replacementProperty>
<property>git.closest.tag.name</property>
<token>^v</token>
<value></value>
<regex>true</regex>
</replacementProperty>
</replacementProperties>
</configuration>
</plugin>
<plugin>
<artifactId>maven-source-plugin</artifactId>
<version>2.1.1</version>
<executions>
<execution>
<id>bundle-sources</id>
<phase>package</phase>
<goals>
<goal>jar-no-fork</goal>
<goal>test-jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.3.1</version>
<executions>
<execution>
<id>resource-bundles</id>
<phase>package</phase>
<goals>
<!-- produce source artifact for main project sources -->
<goal>resource-bundle</goal>
<!-- produce source artifact for project test sources -->
<goal>test-resource-bundle</goal>
</goals>
<configuration>
<detectOfflineLinks>false</detectOfflineLinks>
</configuration>
</execution>
<execution>
<id>javadoc-jar</id>
<phase>package</phase>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
<configuration>
<includeDependencySources>true</includeDependencySources>
<dependencySourceIncludes>
<dependencySourceInclude>it.unimi.dsi:webgraph-big:*</dependencySourceInclude>
</dependencySourceIncludes>
<links>
<link>https://webgraph.di.unimi.it/docs-big/</link>
<link>https://dsiutils.di.unimi.it/docs/</link>
<link>https://fastutil.di.unimi.it/docs/</link>
<link>https://law.di.unimi.it/software/law-docs/</link>
</links>
<tags>
<tag>
<name>implSpec</name>
<placement>a</placement>
<head>Implementation Requirements:</head>
</tag>
<tag>
<name>implNote</name>
<placement>a</placement>
<head>Implementation Note:</head>
</tag>
</tags>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.xolstice.maven.plugins</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
+ <version>0.6.1</version>
+ <configuration>
+ <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+ <pluginId>grpc-java</pluginId>
+ <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>compile</goal>
+ <goal>compile-custom</goal>
+ <goal>test-compile</goal>
+ <goal>test-compile-custom</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
+ <extensions>
+ <extension>
+ <groupId>kr.motd.maven</groupId>
+ <artifactId>os-maven-plugin</artifactId>
+ <version>1.6.2</version>
+ </extension>
+ </extensions>
</build>
</project>
diff --git a/java/src/main/java/org/softwareheritage/graph/rpc/GraphServer.java b/java/src/main/java/org/softwareheritage/graph/rpc/GraphServer.java
new file mode 100644
index 0000000..502e4ac
--- /dev/null
+++ b/java/src/main/java/org/softwareheritage/graph/rpc/GraphServer.java
@@ -0,0 +1,157 @@
+package org.softwareheritage.graph.rpc;
+
+import com.martiansoftware.jsap.*;
+import io.grpc.Server;
+import io.grpc.ServerBuilder;
+import io.grpc.stub.StreamObserver;
+import io.grpc.protobuf.services.ProtoReflectionService;
+import it.unimi.dsi.logging.ProgressLogger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.softwareheritage.graph.SwhBidirectionalGraph;
+import org.softwareheritage.graph.compress.LabelMapBuilder;
+
+import java.io.IOException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Server that manages startup/shutdown of a {@code Greeter} server.
+ */
+public class GraphServer {
+ private final static Logger logger = LoggerFactory.getLogger(GraphServer.class);
+
+ private final SwhBidirectionalGraph graph;
+ private final int port;
+ private final int threads;
+ private Server server;
+
+ public GraphServer(String graphBasename, int port, int threads) throws IOException {
+ this.graph = SwhBidirectionalGraph.loadLabelledMapped(graphBasename, new ProgressLogger(logger));
+ this.port = port;
+ this.threads = threads;
+ graph.loadContentLength();
+ graph.loadContentIsSkipped();
+ graph.loadPersonIds();
+ graph.loadAuthorTimestamps();
+ graph.loadCommitterTimestamps();
+ graph.loadMessages();
+ graph.loadTagNames();
+ graph.loadLabelNames();
+ }
+
+ private void start() throws IOException {
+ server = ServerBuilder.forPort(port).executor(Executors.newFixedThreadPool(threads))
+ .addService(new TraversalService(graph)).addService(ProtoReflectionService.newInstance()).build()
+ .start();
+ logger.info("Server started, listening on " + port);
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ try {
+ GraphServer.this.stop();
+ } catch (InterruptedException e) {
+ e.printStackTrace(System.err);
+ }
+ }));
+ }
+
+ private void stop() throws InterruptedException {
+ if (server != null) {
+ server.shutdown().awaitTermination(30, TimeUnit.SECONDS);
+ }
+ }
+
+ /**
+ * Await termination on the main thread since the grpc library uses daemon threads.
+ */
+ private void blockUntilShutdown() throws InterruptedException {
+ if (server != null) {
+ server.awaitTermination();
+ }
+ }
+
+ private static JSAPResult parseArgs(String[] args) {
+ JSAPResult config = null;
+ try {
+ SimpleJSAP jsap = new SimpleJSAP(LabelMapBuilder.class.getName(), "",
+ new Parameter[]{
+ new FlaggedOption("port", JSAP.INTEGER_PARSER, "50091", JSAP.NOT_REQUIRED, 'p', "port",
+ "The port on which the server should listen."),
+ new FlaggedOption("threads", JSAP.INTEGER_PARSER, "1", JSAP.NOT_REQUIRED, 't', "threads",
+ "The number of concurrent threads. 0 = number of cores."),
+ new UnflaggedOption("graphBasename", JSAP.STRING_PARSER, JSAP.REQUIRED,
+ "Basename of the output graph")});
+
+ config = jsap.parse(args);
+ if (jsap.messagePrinted()) {
+ System.exit(1);
+ }
+ } catch (JSAPException e) {
+ e.printStackTrace();
+ }
+ return config;
+ }
+
+ /**
+ * Main launches the server from the command line.
+ */
+ public static void main(String[] args) throws IOException, InterruptedException {
+ JSAPResult config = parseArgs(args);
+ String graphBasename = config.getString("graphBasename");
+ int port = config.getInt("port");
+ int threads = config.getInt("threads");
+ if (threads == 0) {
+ threads = Runtime.getRuntime().availableProcessors();
+ }
+
+ final GraphServer server = new GraphServer(graphBasename, port, threads);
+ server.start();
+ server.blockUntilShutdown();
+ }
+
+ static class TraversalService extends TraversalServiceGrpc.TraversalServiceImplBase {
+ SwhBidirectionalGraph graph;
+
+ public TraversalService(SwhBidirectionalGraph graph) {
+ this.graph = graph;
+ }
+
+ @Override
+ public void traverse(TraversalRequest request, StreamObserver<Node> responseObserver) {
+ SwhBidirectionalGraph g = graph.copy();
+ Traversal.simpleTraversal(g, request, responseObserver::onNext);
+ responseObserver.onCompleted();
+ }
+
+ @Override
+ public void countNodes(TraversalRequest request, StreamObserver<CountResponse> responseObserver) {
+ AtomicInteger count = new AtomicInteger(0);
+ SwhBidirectionalGraph g = graph.copy();
+ TraversalRequest fixedReq = TraversalRequest.newBuilder(request)
+ // Ignore return fields, just count nodes
+ .setReturnFields(NodeFields.getDefaultInstance()).build();
+ Traversal.simpleTraversal(g, fixedReq, (Node node) -> {
+ count.incrementAndGet();
+ });
+ CountResponse response = CountResponse.newBuilder().setCount(count.get()).build();
+ responseObserver.onNext(response);
+ responseObserver.onCompleted();
+ }
+
+ @Override
+ public void countEdges(TraversalRequest request, StreamObserver<CountResponse> responseObserver) {
+ AtomicInteger count = new AtomicInteger(0);
+ SwhBidirectionalGraph g = graph.copy();
+ TraversalRequest fixedReq = TraversalRequest.newBuilder(request)
+ // Force return empty successors to count the edges
+ .setReturnFields(NodeFields.newBuilder().setSuccessor(true).setSuccessorSwhid(false).build())
+ .build();
+ Traversal.simpleTraversal(g, fixedReq, (Node node) -> {
+ count.addAndGet(node.getSuccessorCount());
+ });
+ CountResponse response = CountResponse.newBuilder().setCount(count.get()).build();
+ responseObserver.onNext(response);
+ responseObserver.onCompleted();
+ }
+ }
+}
diff --git a/java/src/main/java/org/softwareheritage/graph/rpc/Traversal.java b/java/src/main/java/org/softwareheritage/graph/rpc/Traversal.java
new file mode 100644
index 0000000..e046476
--- /dev/null
+++ b/java/src/main/java/org/softwareheritage/graph/rpc/Traversal.java
@@ -0,0 +1,275 @@
+package org.softwareheritage.graph.rpc;
+
+import com.google.protobuf.ByteString;
+import it.unimi.dsi.big.webgraph.LazyLongIterator;
+import it.unimi.dsi.big.webgraph.labelling.ArcLabelledNodeIterator;
+import it.unimi.dsi.big.webgraph.labelling.Label;
+import org.softwareheritage.graph.*;
+import org.softwareheritage.graph.labels.DirEntry;
+
+import java.util.*;
+
+public class Traversal {
+ private static LazyLongIterator filterSuccessors(SwhUnidirectionalGraph g, long nodeId, AllowedEdges allowedEdges) {
+ if (allowedEdges.restrictedTo == null) {
+ // All edges are allowed, bypass edge check
+ return g.successors(nodeId);
+ } else {
+ LazyLongIterator allSuccessors = g.successors(nodeId);
+ return new LazyLongIterator() {
+ @Override
+ public long nextLong() {
+ long neighbor;
+ while ((neighbor = allSuccessors.nextLong()) != -1) {
+ if (allowedEdges.isAllowed(g.getNodeType(nodeId), g.getNodeType(neighbor))) {
+ return neighbor;
+ }
+ }
+ return -1;
+ }
+
+ @Override
+ public long skip(final long n) {
+ long i = 0;
+ while (i < n && nextLong() != -1)
+ i++;
+ return i;
+ }
+ };
+ }
+ }
+
+ private static ArcLabelledNodeIterator.LabelledArcIterator filterLabelledSuccessors(SwhUnidirectionalGraph g,
+ long nodeId, AllowedEdges allowedEdges) {
+ if (allowedEdges.restrictedTo == null) {
+ // All edges are allowed, bypass edge check
+ return g.labelledSuccessors(nodeId);
+ } else {
+ ArcLabelledNodeIterator.LabelledArcIterator allSuccessors = g.labelledSuccessors(nodeId);
+ return new ArcLabelledNodeIterator.LabelledArcIterator() {
+ @Override
+ public Label label() {
+ return allSuccessors.label();
+ }
+
+ @Override
+ public long nextLong() {
+ long neighbor;
+ while ((neighbor = allSuccessors.nextLong()) != -1) {
+ if (allowedEdges.isAllowed(g.getNodeType(nodeId), g.getNodeType(neighbor))) {
+ return neighbor;
+ }
+ }
+ return -1;
+ }
+
+ @Override
+ public long skip(final long n) {
+ long i = 0;
+ while (i < n && nextLong() != -1)
+ i++;
+ return i;
+ }
+ };
+ }
+ }
+
+ private static class NodeFilterChecker {
+ private final SwhUnidirectionalGraph g;
+ private final NodeFilter filter;
+ private final AllowedNodes allowedNodes;
+
+ private NodeFilterChecker(SwhUnidirectionalGraph graph, NodeFilter filter) {
+ this.g = graph;
+ this.filter = filter;
+ this.allowedNodes = new AllowedNodes(filter.hasTypes() ? filter.getTypes() : "*");
+ }
+
+ public boolean allowed(long nodeId) {
+ if (filter == null) {
+ return true;
+ }
+ if (!this.allowedNodes.isAllowed(g.getNodeType(nodeId))) {
+ return false;
+ }
+
+ long outdegree = g.outdegree(nodeId);
+ if (filter.hasMinTraversalSuccessors() && outdegree < filter.getMinTraversalSuccessors()) {
+ return false;
+ }
+ if (filter.hasMaxTraversalSuccessors() && outdegree > filter.getMaxTraversalSuccessors()) {
+ return false;
+ }
+
+ return true;
+ }
+ }
+
+ public static SwhUnidirectionalGraph getDirectedGraph(SwhBidirectionalGraph g, TraversalRequest request) {
+ switch (request.getDirection()) {
+ case FORWARD:
+ return g.getForwardGraph();
+ case BACKWARD:
+ return g.getBackwardGraph();
+ case BOTH:
+ return new SwhUnidirectionalGraph(g.symmetrize(), g.getProperties());
+ }
+ throw new IllegalArgumentException("Unknown direction: " + request.getDirection());
+ }
+
+ public static void simpleTraversal(SwhBidirectionalGraph bidirectionalGraph, TraversalRequest request,
+ NodeObserver nodeObserver) {
+ SwhUnidirectionalGraph g = getDirectedGraph(bidirectionalGraph, request);
+ NodeFilterChecker nodeReturnChecker = new NodeFilterChecker(g, request.getReturnNodes());
+
+ AllowedEdges allowedEdges = new AllowedEdges(request.hasEdges() ? request.getEdges() : "*");
+
+ Queue<Long> queue = new ArrayDeque<>();
+ HashSet<Long> visited = new HashSet<>();
+ request.getSrcList().forEach(srcSwhid -> {
+ long srcNodeId = g.getNodeId(new SWHID(srcSwhid));
+ queue.add(srcNodeId);
+ visited.add(srcNodeId);
+ });
+ queue.add(-1L); // Depth sentinel
+
+ long edgesAccessed = 0;
+ long currentDepth = 0;
+ while (!queue.isEmpty()) {
+ long curr = queue.poll();
+ if (curr == -1L) {
+ ++currentDepth;
+ if (!queue.isEmpty()) {
+ queue.add(-1L);
+ }
+ continue;
+ }
+ if (request.hasMaxDepth() && currentDepth > request.getMaxDepth()) {
+ break;
+ }
+ edgesAccessed += g.outdegree(curr);
+ if (request.hasMaxEdges() && edgesAccessed >= request.getMaxEdges()) {
+ break;
+ }
+
+ Node.Builder nodeBuilder = null;
+ if (nodeReturnChecker.allowed(curr)) {
+ nodeBuilder = Node.newBuilder();
+ buildNodeProperties(g, request.getReturnFields(), nodeBuilder, curr);
+ }
+
+ ArcLabelledNodeIterator.LabelledArcIterator it = filterLabelledSuccessors(g, curr, allowedEdges);
+ for (long succ; (succ = it.nextLong()) != -1;) {
+ if (!visited.contains(succ)) {
+ queue.add(succ);
+ visited.add(succ);
+ }
+ buildSuccessorProperties(g, request.getReturnFields(), nodeBuilder, curr, succ, it.label());
+ }
+ if (nodeBuilder != null) {
+ nodeObserver.onNext(nodeBuilder.build());
+ }
+ }
+ }
+
+ private static void buildNodeProperties(SwhUnidirectionalGraph graph, NodeFields fields, Node.Builder nodeBuilder,
+ long node) {
+ if (fields == null || !fields.hasSwhid() || fields.getSwhid()) {
+ nodeBuilder.setSwhid(graph.getSWHID(node).toString());
+ }
+ if (fields == null) {
+ return;
+ }
+
+ switch (graph.getNodeType(node)) {
+ case CNT:
+ if (fields.hasCntLength()) {
+ nodeBuilder.setCntLength(graph.getContentLength(node));
+ }
+ if (fields.hasCntIsSkipped()) {
+ nodeBuilder.setCntIsSkipped(graph.isContentSkipped(node));
+ }
+ break;
+ case REV:
+ if (fields.getRevAuthor()) {
+ nodeBuilder.setRevAuthor(graph.getAuthorId(node));
+ }
+ if (fields.getRevCommitter()) {
+ nodeBuilder.setRevAuthor(graph.getCommitterId(node));
+ }
+ if (fields.getRevAuthorDate()) {
+ nodeBuilder.setRevAuthorDate(graph.getAuthorTimestamp(node));
+ }
+ if (fields.getRevAuthorDateOffset()) {
+ nodeBuilder.setRevAuthorDateOffset(graph.getAuthorTimestampOffset(node));
+ }
+ if (fields.getRevCommitterDate()) {
+ nodeBuilder.setRevCommitterDate(graph.getCommitterTimestamp(node));
+ }
+ if (fields.getRevCommitterDateOffset()) {
+ nodeBuilder.setRevCommitterDateOffset(graph.getCommitterTimestampOffset(node));
+ }
+ if (fields.getRevMessage()) {
+ byte[] msg = graph.getMessage(node);
+ if (msg != null) {
+ nodeBuilder.setRevMessage(ByteString.copyFrom(msg));
+ }
+ }
+ break;
+ case REL:
+ if (fields.getRelAuthor()) {
+ nodeBuilder.setRelAuthor(graph.getAuthorId(node));
+ }
+ if (fields.getRelAuthorDate()) {
+ nodeBuilder.setRelAuthorDate(graph.getAuthorTimestamp(node));
+ }
+ if (fields.getRelAuthorDateOffset()) {
+ nodeBuilder.setRelAuthorDateOffset(graph.getAuthorTimestampOffset(node));
+ }
+ if (fields.getRelName()) {
+ byte[] msg = graph.getTagName(node);
+ if (msg != null) {
+ nodeBuilder.setRelName(ByteString.copyFrom(msg));
+ }
+ }
+ if (fields.getRelMessage()) {
+ byte[] msg = graph.getMessage(node);
+ if (msg != null) {
+ nodeBuilder.setRelMessage(ByteString.copyFrom(msg));
+ }
+ }
+ break;
+ case ORI:
+ if (fields.getOriUrl()) {
+ String url = graph.getUrl(node);
+ if (url != null) {
+ nodeBuilder.setOriUrl(url);
+ }
+ }
+ }
+ }
+
+ private static void buildSuccessorProperties(SwhUnidirectionalGraph graph, NodeFields fields,
+ Node.Builder nodeBuilder, long src, long dst, Label label) {
+ if (nodeBuilder != null && fields != null && fields.getSuccessor()) {
+ Successor.Builder successorBuilder = Successor.newBuilder();
+ if (!fields.hasSuccessorSwhid() || fields.getSuccessorSwhid()) {
+ successorBuilder.setSwhid(graph.getSWHID(dst).toString());
+ }
+ if (fields.getSuccessorLabel()) {
+ DirEntry[] entries = (DirEntry[]) label.get();
+ for (DirEntry entry : entries) {
+ EdgeLabel.Builder builder = EdgeLabel.newBuilder();
+ builder.setName(ByteString.copyFrom(graph.getLabelName(entry.filenameId)));
+ builder.setPermission(entry.permission);
+ successorBuilder.addLabel(builder.build());
+ }
+ }
+ nodeBuilder.addSuccessor(successorBuilder.build());
+ }
+ }
+
+ public interface NodeObserver {
+ void onNext(Node nodeId);
+ }
+}
diff --git a/proto/swhgraph.proto b/proto/swhgraph.proto
new file mode 100644
index 0000000..0986234
--- /dev/null
+++ b/proto/swhgraph.proto
@@ -0,0 +1,102 @@
+syntax = "proto3";
+
+option java_multiple_files = true;
+option java_package = "org.softwareheritage.graph.rpc";
+option java_outer_classname = "GraphService";
+
+package swh.graph;
+
+service TraversalService {
+ rpc Traverse (TraversalRequest) returns (stream Node);
+ rpc CountNodes (TraversalRequest) returns (CountResponse);
+ rpc CountEdges (TraversalRequest) returns (CountResponse);
+}
+
+enum GraphDirection {
+ FORWARD = 0;
+ BACKWARD = 1;
+ BOTH = 2;
+}
+
+message TraversalRequest {
+ repeated string src = 1;
+
+ // Traversal options
+ optional GraphDirection direction = 2;
+ optional string edges = 3;
+ optional int64 max_edges = 4;
+ optional int64 max_depth = 5;
+ optional NodeFilter return_nodes = 6;
+ optional NodeFields return_fields = 7;
+}
+
+message NodeFilter {
+ optional string types = 1;
+ optional int64 min_traversal_successors = 2;
+ optional int64 max_traversal_successors = 3;
+}
+
+message NodeFields {
+ optional bool swhid = 1;
+
+ optional bool successor = 2;
+ optional bool successor_swhid = 3;
+ optional bool successor_label = 4;
+
+ optional bool cnt_length = 5;
+ optional bool cnt_is_skipped = 6;
+
+ optional bool rev_author = 7;
+ optional bool rev_author_date = 8;
+ optional bool rev_author_date_offset = 9;
+ optional bool rev_committer = 10;
+ optional bool rev_committer_date = 11;
+ optional bool rev_committer_date_offset = 12;
+ optional bool rev_message = 13;
+
+ optional bool rel_author = 14;
+ optional bool rel_author_date = 15;
+ optional bool rel_author_date_offset = 16;
+ optional bool rel_name = 17;
+ optional bool rel_message = 18;
+
+ optional bool ori_url = 19;
+}
+
+message Node {
+ string swhid = 1;
+ repeated Successor successor = 2;
+
+ optional int64 cnt_length = 3;
+ optional bool cnt_is_skipped = 4;
+
+ optional int64 rev_author = 5;
+ optional int64 rev_author_date = 6;
+ optional int32 rev_author_date_offset = 7;
+ optional int64 rev_committer = 8;
+ optional int64 rev_committer_date = 9;
+ optional int32 rev_committer_date_offset = 10;
+ optional bytes rev_message = 11;
+
+ optional int64 rel_author = 12;
+ optional int64 rel_author_date = 13;
+ optional int32 rel_author_date_offset = 14;
+ optional bytes rel_name = 15;
+ optional bytes rel_message = 16;
+
+ optional string ori_url = 17;
+}
+
+message Successor {
+ optional string swhid = 1;
+ repeated EdgeLabel label = 2;
+}
+
+message EdgeLabel {
+ bytes name = 1;
+ int32 permission = 2;
+}
+
+message CountResponse {
+ int64 count = 1;
+}