diff --git a/api/server/src/main/java/org/softwareheritage/graph/App.java b/api/server/src/main/java/org/softwareheritage/graph/App.java index a127d75..01156f7 100644 --- a/api/server/src/main/java/org/softwareheritage/graph/App.java +++ b/api/server/src/main/java/org/softwareheritage/graph/App.java @@ -1,88 +1,104 @@ package org.softwareheritage.graph; import java.io.IOException; import java.util.List; import java.util.Map; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; import io.javalin.Javalin; +import io.javalin.http.Context; import io.javalin.plugin.json.JavalinJackson; import org.softwareheritage.graph.Graph; import org.softwareheritage.graph.SwhId; import org.softwareheritage.graph.algo.Stats; import org.softwareheritage.graph.algo.Visit; +import org.softwareheritage.graph.algo.Walk; public class App { public static void main(String[] args) throws IOException { String path = args[0]; Graph graph = new Graph(path); Stats stats = new Stats(path); // Clean up on exit Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { try { graph.cleanUp(); } catch (IOException e) { System.out.println("Could not clean up graph on exit: " + e); } } }); // Configure Jackson JSON to use snake case naming style ObjectMapper objectMapper = JavalinJackson.getObjectMapper(); objectMapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); JavalinJackson.configure(objectMapper); Javalin app = Javalin.create().start(5009); - // Check query strings on API endpoints - app.before("/visit/*", ctx -> { - Map> queryParamMap = ctx.queryParamMap(); - for (String key : queryParamMap.keySet()) { - if (!key.matches("direction|edges")) { - throw new IllegalArgumentException("Unknown query string: " + key); - } - } - }); + app.before("/stats/*", ctx -> { checkQueryStrings(ctx, ""); }); + app.before("/visit/*", ctx -> { checkQueryStrings(ctx, "direction|edges"); }); + app.before("/walk/*", ctx -> { checkQueryStrings(ctx, "direction|edges|traversal"); }); app.get("/stats/", ctx -> { ctx.json(stats); }); // Graph traversal endpoints // By default the traversal is a forward DFS using all edges app.get("/visit/:src", ctx -> { SwhId src = new SwhId(ctx.pathParam("src")); String direction = ctx.queryParam("direction", "forward"); - String edges = ctx.queryParam("edges", "*"); + String edgesFmt = ctx.queryParam("edges", "*"); - Visit visit = new Visit(graph, src, edges, direction, Visit.OutputFmt.NODES_AND_PATHS); + Visit visit = new Visit(graph, src, edgesFmt, direction, Visit.OutputFmt.NODES_AND_PATHS); ctx.json(visit); }); app.get("/visit/nodes/:src", ctx -> { SwhId src = new SwhId(ctx.pathParam("src")); String direction = ctx.queryParam("direction", "forward"); - String edges = ctx.queryParam("edges", "*"); + String edgesFmt = ctx.queryParam("edges", "*"); - Visit visit = new Visit(graph, src, edges, direction, Visit.OutputFmt.ONLY_NODES); + Visit visit = new Visit(graph, src, edgesFmt, direction, Visit.OutputFmt.ONLY_NODES); ctx.json(visit.getNodes()); }); app.get("/visit/paths/:src", ctx -> { SwhId src = new SwhId(ctx.pathParam("src")); String direction = ctx.queryParam("direction", "forward"); - String edges = ctx.queryParam("edges", "*"); + String edgesFmt = ctx.queryParam("edges", "*"); - Visit visit = new Visit(graph, src, edges, direction, Visit.OutputFmt.ONLY_PATHS); + Visit visit = new Visit(graph, src, edgesFmt, direction, Visit.OutputFmt.ONLY_PATHS); ctx.json(visit.getPaths()); }); + app.get("/walk/:src/:dst", ctx -> { + SwhId src = new SwhId(ctx.pathParam("src")); + String dstFmt = ctx.pathParam("dst"); + String direction = ctx.queryParam("direction", "forward"); + String edgesFmt = ctx.queryParam("edges", "*"); + String traversal = ctx.queryParam("traversal", "dfs"); + + Walk walk = new Walk(graph, src, dstFmt, edgesFmt, direction, traversal); + ctx.json(walk.getPath()); + }); + app.exception(IllegalArgumentException.class, (e, ctx) -> { ctx.status(400); ctx.result(e.getMessage()); }); } + + private static void checkQueryStrings(Context ctx, String allowedFmt) { + Map> queryParamMap = ctx.queryParamMap(); + for (String key : queryParamMap.keySet()) { + if (!key.matches(allowedFmt)) { + throw new IllegalArgumentException("Unknown query string: " + key); + } + } + } } diff --git a/api/server/src/main/java/org/softwareheritage/graph/SwhPath.java b/api/server/src/main/java/org/softwareheritage/graph/SwhPath.java index be9c80b..e1927b2 100644 --- a/api/server/src/main/java/org/softwareheritage/graph/SwhPath.java +++ b/api/server/src/main/java/org/softwareheritage/graph/SwhPath.java @@ -1,76 +1,81 @@ package org.softwareheritage.graph; import java.util.ArrayList; +import java.util.Collections; import com.fasterxml.jackson.annotation.JsonValue; import org.softwareheritage.graph.SwhId; public class SwhPath { ArrayList path; public SwhPath() { this.path = new ArrayList(); } public SwhPath(String ...swhIds) { this(); for (String swhId : swhIds) { add(new SwhId(swhId)); } } public SwhPath(SwhId ...swhIds) { this(); for (SwhId swhId : swhIds) { add(swhId); } } @JsonValue public ArrayList getPath() { return path; } public void add(SwhId swhId) { path.add(swhId); } public SwhId get(int index) { return path.get(index); } public int size() { return path.size(); } + public void reverse() { + Collections.reverse(path); + } + @Override public boolean equals(Object otherObj) { if (otherObj == this) return true; if (!(otherObj instanceof SwhPath)) return false; SwhPath other = (SwhPath) otherObj; if (size() != other.size()) { return false; } for (int i = 0; i < size(); i++) { SwhId thisSwhId = get(i); SwhId otherSwhId = other.get(i); if (!thisSwhId.equals(otherSwhId)) { return false; } } return true; } @Override public String toString() { String str = new String(); for (SwhId swhId : path) { str += swhId + "/"; } return str; } } diff --git a/api/server/src/main/java/org/softwareheritage/graph/algo/Visit.java b/api/server/src/main/java/org/softwareheritage/graph/algo/Visit.java index 817f4a9..d69cf62 100644 --- a/api/server/src/main/java/org/softwareheritage/graph/algo/Visit.java +++ b/api/server/src/main/java/org/softwareheritage/graph/algo/Visit.java @@ -1,97 +1,98 @@ package org.softwareheritage.graph.algo; import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.Stack; import it.unimi.dsi.big.webgraph.LazyLongIterator; import org.softwareheritage.graph.Edges; import org.softwareheritage.graph.Graph; import org.softwareheritage.graph.Node; import org.softwareheritage.graph.SwhId; import org.softwareheritage.graph.SwhPath; public class Visit { public enum OutputFmt { ONLY_NODES, ONLY_PATHS, NODES_AND_PATHS } Graph graph; boolean useTransposed; Edges edges; + // LinkedHashSet is necessary to preserve insertion order LinkedHashSet nodes; ArrayList paths; Stack currentPath; - public Visit(Graph graph, SwhId src, String edges, String direction, OutputFmt output) { + public Visit(Graph graph, SwhId src, String edgesFmt, String direction, OutputFmt output) { if (!direction.matches("forward|backward")) { throw new IllegalArgumentException("Unknown traversal direction: " + direction); } this.graph = graph; this.useTransposed = (direction.equals("backward")); - this.edges = new Edges(edges); + this.edges = new Edges(edgesFmt); this.nodes = new LinkedHashSet(); this.paths = new ArrayList(); this.currentPath = new Stack(); long nodeId = graph.getNodeId(src); if (output == OutputFmt.ONLY_NODES) { dfsOutputOnlyNodes(nodeId); } else { dfs(nodeId); } } public LinkedHashSet getNodes() { return nodes; } public ArrayList getPaths() { return paths; } private void dfs(long currentNodeId) { nodes.add(graph.getSwhId(currentNodeId)); currentPath.push(currentNodeId); long degree = graph.degree(currentNodeId, useTransposed); LazyLongIterator neighbors = graph.neighbors(currentNodeId, useTransposed); long visitedNeighbors = 0; while (degree-- > 0) { long neighborNodeId = neighbors.nextLong(); Node.Type currentNodeType = graph.getSwhId(currentNodeId).getType(); Node.Type neighborNodeType = graph.getSwhId(neighborNodeId).getType(); if (edges.isAllowed(currentNodeType, neighborNodeType)) { dfs(neighborNodeId); visitedNeighbors++; } } if (visitedNeighbors == 0) { SwhPath path = new SwhPath(); for (long nodeId : currentPath) { path.add(graph.getSwhId(nodeId)); } paths.add(path); } currentPath.pop(); } private void dfsOutputOnlyNodes(long currentNodeId) { nodes.add(graph.getSwhId(currentNodeId)); long degree = graph.degree(currentNodeId, useTransposed); LazyLongIterator neighbors = graph.neighbors(currentNodeId, useTransposed); while (degree-- > 0) { long neighborNodeId = neighbors.nextLong(); Node.Type currentNodeType = graph.getSwhId(currentNodeId).getType(); Node.Type neighborNodeType = graph.getSwhId(neighborNodeId).getType(); if (edges.isAllowed(currentNodeType, neighborNodeType)) { dfsOutputOnlyNodes(neighborNodeId); } } } } diff --git a/api/server/src/main/java/org/softwareheritage/graph/algo/Walk.java b/api/server/src/main/java/org/softwareheritage/graph/algo/Walk.java new file mode 100644 index 0000000..71ba73b --- /dev/null +++ b/api/server/src/main/java/org/softwareheritage/graph/algo/Walk.java @@ -0,0 +1,148 @@ +package org.softwareheritage.graph.algo; + +import java.util.Stack; +import java.util.Queue; +import java.util.LinkedList; + +import it.unimi.dsi.big.webgraph.LazyLongIterator; +import it.unimi.dsi.bits.LongArrayBitVector; +import it.unimi.dsi.fastutil.longs.LongBigArrays; + +import org.softwareheritage.graph.Edges; +import org.softwareheritage.graph.Graph; +import org.softwareheritage.graph.Node; +import org.softwareheritage.graph.SwhId; +import org.softwareheritage.graph.SwhPath; + +public class Walk { + Graph graph; + boolean useTransposed; + Edges edges; + + SwhPath path; + long[][] nodeParent; + + public Walk( + Graph graph, SwhId src, String dstFmt, String edgesFmt, String direction, String traversal) { + if (!traversal.matches("dfs|bfs")) { + throw new IllegalArgumentException("Unknown traversal algorithm: " + traversal); + } + if (!direction.matches("forward|backward")) { + throw new IllegalArgumentException("Unknown traversal direction: " + direction); + } + + this.graph = graph; + this.useTransposed = (direction.equals("backward")); + this.edges = new Edges(edgesFmt); + this.path = new SwhPath(); + this.nodeParent = LongBigArrays.newBigArray(graph.getNbNodes()); + LongBigArrays.fill(nodeParent, -1); + + long srcNodeId = graph.getNodeId(src); + long dstNodeId; + if (traversal.equals("dfs")) { + dstNodeId = dfs(srcNodeId, dstFmt); + } else { + dstNodeId = bfs(srcNodeId, dstFmt); + } + + if (dstNodeId == -1) { + throw new IllegalArgumentException("Unable to find destination point: " + dstFmt); + } else { + backtracking(srcNodeId, dstNodeId); + } + } + + public SwhPath getPath() { + return path; + } + + private void backtracking(long srcNodeId, long dstNodeId) { + long currentNodeId = dstNodeId; + while (currentNodeId != srcNodeId) { + path.add(graph.getSwhId(currentNodeId)); + currentNodeId = LongBigArrays.get(nodeParent, currentNodeId); + } + path.add(graph.getSwhId(srcNodeId)); + path.reverse(); + } + + private long dfs(long srcNodeId, String dstFmt) { + Stack stack = new Stack(); + LongArrayBitVector visited = LongArrayBitVector.ofLength(graph.getNbNodes()); + + stack.push(srcNodeId); + visited.set(srcNodeId); + + while (!stack.isEmpty()) { + long currentNodeId = stack.pop(); + SwhId currentSwhId = graph.getSwhId(currentNodeId); + if (isDestinationNode(currentSwhId, dstFmt)) { + return currentNodeId; + } + + long degree = graph.degree(currentNodeId, useTransposed); + LazyLongIterator neighbors = graph.neighbors(currentNodeId, useTransposed); + + while (degree-- > 0) { + long neighborNodeId = neighbors.nextLong(); + SwhId neighborSwhId = graph.getSwhId(neighborNodeId); + if (!visited.getBoolean(neighborNodeId) + && edges.isAllowed(currentSwhId.getType(), neighborSwhId.getType())) { + stack.push(neighborNodeId); + visited.set(neighborNodeId); + LongBigArrays.set(nodeParent, neighborNodeId, currentNodeId); + } + } + } + + return -1; + } + + private long bfs(long srcNodeId, String dstFmt) { + Queue queue = new LinkedList(); + LongArrayBitVector visited = LongArrayBitVector.ofLength(graph.getNbNodes()); + + queue.add(srcNodeId); + visited.set(srcNodeId); + + while (!queue.isEmpty()) { + long currentNodeId = queue.poll(); + SwhId currentSwhId = graph.getSwhId(currentNodeId); + if (isDestinationNode(currentSwhId, dstFmt)) { + return currentNodeId; + } + + long degree = graph.degree(currentNodeId, useTransposed); + LazyLongIterator neighbors = graph.neighbors(currentNodeId, useTransposed); + + while (degree-- > 0) { + long neighborNodeId = neighbors.nextLong(); + SwhId neighborSwhId = graph.getSwhId(neighborNodeId); + if (!visited.getBoolean(neighborNodeId) + && edges.isAllowed(currentSwhId.getType(), neighborSwhId.getType())) { + queue.add(neighborNodeId); + visited.set(neighborNodeId); + LongBigArrays.set(nodeParent, neighborNodeId, currentNodeId); + } + } + } + + return -1; + } + + private boolean isDestinationNode(SwhId swhId, String dstFmt) { + // dstFmt is either a SwhId... + if (swhId.toString().equals(dstFmt)) { + return true; + } + + // ...or a Node.Type + try { + Node.Type dstType = Node.Type.fromStr(dstFmt); + return (swhId.getType().equals(dstType)); + } catch (IllegalArgumentException e) { + return false; + } + } +}