diff --git a/api/server/src/main/java/org/softwareheritage/graph/Graph.java b/api/server/src/main/java/org/softwareheritage/graph/Graph.java index 8143c9b..d015f37 100644 --- a/api/server/src/main/java/org/softwareheritage/graph/Graph.java +++ b/api/server/src/main/java/org/softwareheritage/graph/Graph.java @@ -1,71 +1,70 @@ package org.softwareheritage.graph; import java.io.IOException; import it.unimi.dsi.big.webgraph.BVGraph; -import it.unimi.dsi.big.webgraph.LazyLongIterator; import org.softwareheritage.graph.SwhId; import org.softwareheritage.graph.backend.NodeIdMap; public class Graph { BVGraph graph; BVGraph graphTransposed; String path; NodeIdMap nodeIdMap; public Graph(String path) throws IOException { this.graph = BVGraph.load(path); this.graphTransposed = BVGraph.load(path + "-transposed"); this.path = path; this.nodeIdMap = new NodeIdMap(path, getNbNodes()); } public void cleanUp() throws IOException { nodeIdMap.close(); } public String getPath() { return path; } public long getNodeId(SwhId swhId) { return nodeIdMap.getNodeId(swhId); } public SwhId getSwhId(long nodeId) { return nodeIdMap.getSwhId(nodeId); } public long getNbNodes() { return graph.numNodes(); } public long getNbEdges() { return graph.numArcs(); } - public LazyLongIterator successors(long nodeId) { - return graph.successors(nodeId); + public long[][] successors(long nodeId) { + return graph.successorBigArray(nodeId); } public long outdegree(long nodeId) { return graph.outdegree(nodeId); } - public LazyLongIterator predecessors(long nodeId) { - return graphTransposed.successors(nodeId); + public long[][] predecessors(long nodeId) { + return graphTransposed.successorBigArray(nodeId); } public long indegree(long nodeId) { return graphTransposed.outdegree(nodeId); } public long degree(long nodeId, boolean useTransposed) { return (useTransposed) ? indegree(nodeId) : outdegree(nodeId); } - public LazyLongIterator neighbors(long nodeId, boolean useTransposed) { + public long[][] neighbors(long nodeId, boolean useTransposed) { return (useTransposed) ? predecessors(nodeId) : successors(nodeId); } } diff --git a/api/server/src/main/java/org/softwareheritage/graph/Neighbors.java b/api/server/src/main/java/org/softwareheritage/graph/Neighbors.java new file mode 100644 index 0000000..7fcf609 --- /dev/null +++ b/api/server/src/main/java/org/softwareheritage/graph/Neighbors.java @@ -0,0 +1,60 @@ +package org.softwareheritage.graph; + +import java.util.Iterator; + +import it.unimi.dsi.fastutil.longs.LongBigArrays; + +import org.softwareheritage.graph.AllowedEdges; +import org.softwareheritage.graph.Graph; +import org.softwareheritage.graph.SwhId; + +public class Neighbors implements Iterable { + Graph graph; + boolean useTransposed; + AllowedEdges edges; + SwhId src; + + public Neighbors(Graph graph, boolean useTransposed, AllowedEdges edges, SwhId src) { + this.graph = graph; + this.useTransposed = useTransposed; + this.edges = edges; + this.src = src; + } + + @Override + public Iterator iterator() { + return new NeighborsIterator(); + } + + public class NeighborsIterator implements Iterator { + long nextNeighborIdx; + long nbNeighbors; + long[][] neighbors; + + public NeighborsIterator() { + long srcNodeId = graph.getNodeId(src); + + this.nextNeighborIdx = -1; + this.nbNeighbors = graph.degree(srcNodeId, useTransposed); + this.neighbors = graph.neighbors(srcNodeId, useTransposed); + } + + // Look ahead because with edge restriction not all neighbors are considered + public boolean hasNext() { + for (long lookAheadIdx = nextNeighborIdx + 1; lookAheadIdx < nbNeighbors; lookAheadIdx++) { + long nextNodeId = LongBigArrays.get(neighbors, lookAheadIdx); + SwhId nextSwhId = graph.getSwhId(nextNodeId); + if (edges.isAllowed(src.getType(), nextSwhId.getType())) { + nextNeighborIdx = lookAheadIdx; + return true; + } + } + return false; + } + + public Long next() { + long nextNodeId = LongBigArrays.get(neighbors, nextNeighborIdx); + return nextNodeId; + } + } +} diff --git a/api/server/src/main/java/org/softwareheritage/graph/algo/Leaves.java b/api/server/src/main/java/org/softwareheritage/graph/algo/Leaves.java index 40b9062..53a78ce 100644 --- a/api/server/src/main/java/org/softwareheritage/graph/algo/Leaves.java +++ b/api/server/src/main/java/org/softwareheritage/graph/algo/Leaves.java @@ -1,64 +1,55 @@ package org.softwareheritage.graph.algo; import java.util.ArrayList; -import it.unimi.dsi.big.webgraph.LazyLongIterator; import it.unimi.dsi.bits.LongArrayBitVector; import org.softwareheritage.graph.AllowedEdges; import org.softwareheritage.graph.Graph; -import org.softwareheritage.graph.Node; +import org.softwareheritage.graph.Neighbors; import org.softwareheritage.graph.SwhId; public class Leaves { Graph graph; boolean useTransposed; AllowedEdges edges; ArrayList leaves; LongArrayBitVector visited; public Leaves(Graph graph, SwhId src, String edgesFmt, String direction) { if (!direction.matches("forward|backward")) { throw new IllegalArgumentException("Unknown traversal direction: " + direction); } this.graph = graph; this.useTransposed = (direction.equals("backward")); this.edges = new AllowedEdges(edgesFmt); this.leaves = new ArrayList(); this.visited = LongArrayBitVector.ofLength(graph.getNbNodes()); long nodeId = graph.getNodeId(src); dfs(nodeId); } public ArrayList getLeaves() { return leaves; } private void dfs(long currentNodeId) { visited.set(currentNodeId); SwhId currentSwhId = graph.getSwhId(currentNodeId); - long degree = graph.degree(currentNodeId, useTransposed); - LazyLongIterator neighbors = graph.neighbors(currentNodeId, useTransposed); long neighborsCnt = 0; - - while (degree-- > 0) { - long neighborNodeId = neighbors.nextLong(); - Node.Type currentNodeType = currentSwhId.getType(); - Node.Type neighborNodeType = graph.getSwhId(neighborNodeId).getType(); - if (edges.isAllowed(currentNodeType, neighborNodeType)) { - neighborsCnt++; - if (!visited.getBoolean(neighborNodeId)) { - dfs(neighborNodeId); - } + for (long neighborNodeId : new Neighbors(graph, useTransposed, edges, currentSwhId)) { + neighborsCnt++; + if (!visited.getBoolean(neighborNodeId)) { + dfs(neighborNodeId); } } if (neighborsCnt == 0) { leaves.add(currentSwhId); } } } diff --git a/api/server/src/main/java/org/softwareheritage/graph/algo/Neighbors.java b/api/server/src/main/java/org/softwareheritage/graph/algo/Neighbors.java index 133d110..9f38441 100644 --- a/api/server/src/main/java/org/softwareheritage/graph/algo/Neighbors.java +++ b/api/server/src/main/java/org/softwareheritage/graph/algo/Neighbors.java @@ -1,49 +1,39 @@ package org.softwareheritage.graph.algo; import java.util.ArrayList; -import it.unimi.dsi.big.webgraph.LazyLongIterator; - import org.softwareheritage.graph.AllowedEdges; import org.softwareheritage.graph.Graph; import org.softwareheritage.graph.SwhId; public class Neighbors { Graph graph; boolean useTransposed; AllowedEdges edges; ArrayList neighbors; public Neighbors(Graph graph, SwhId src, String edgesFmt, String direction) { if (!direction.matches("forward|backward")) { throw new IllegalArgumentException("Unknown traversal direction: " + direction); } this.graph = graph; this.useTransposed = (direction.equals("backward")); this.edges = new AllowedEdges(edgesFmt); this.neighbors = new ArrayList(); iterateNeighbors(src); } public ArrayList getNeighbors() { return neighbors; } private void iterateNeighbors(SwhId swhId) { - long nodeId = graph.getNodeId(swhId); - long degree = graph.degree(nodeId, useTransposed); - LazyLongIterator neighborsNodeIds = graph.neighbors(nodeId, useTransposed); - - while (degree-- > 0) { - long neighborNodeId = neighborsNodeIds.nextLong(); - SwhId neighborSwhId = graph.getSwhId(neighborNodeId); - if (edges.isAllowed(swhId.getType(), neighborSwhId.getType())) { - neighbors.add(neighborSwhId); - } + // TEMPORARY FIX: Avoid import naming problem with Neighbors + for (long neighborNodeId : new org.softwareheritage.graph.Neighbors(graph, useTransposed, edges, swhId)) { + neighbors.add(graph.getSwhId(neighborNodeId)); } } } - diff --git a/api/server/src/main/java/org/softwareheritage/graph/algo/Visit.java b/api/server/src/main/java/org/softwareheritage/graph/algo/Visit.java index c1118ee..b57ba01 100644 --- a/api/server/src/main/java/org/softwareheritage/graph/algo/Visit.java +++ b/api/server/src/main/java/org/softwareheritage/graph/algo/Visit.java @@ -1,104 +1,90 @@ package org.softwareheritage.graph.algo; import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.Stack; -import it.unimi.dsi.big.webgraph.LazyLongIterator; import it.unimi.dsi.bits.LongArrayBitVector; import org.softwareheritage.graph.AllowedEdges; import org.softwareheritage.graph.Graph; -import org.softwareheritage.graph.Node; +import org.softwareheritage.graph.Neighbors; import org.softwareheritage.graph.SwhId; import org.softwareheritage.graph.SwhPath; public class Visit { public enum OutputFmt { ONLY_NODES, ONLY_PATHS, NODES_AND_PATHS } Graph graph; boolean useTransposed; AllowedEdges edges; // LinkedHashSet is necessary to preserve insertion order LinkedHashSet nodes; ArrayList paths; Stack currentPath; LongArrayBitVector visited; public Visit(Graph graph, SwhId src, String edgesFmt, String direction, OutputFmt output) { if (!direction.matches("forward|backward")) { throw new IllegalArgumentException("Unknown traversal direction: " + direction); } this.graph = graph; this.useTransposed = (direction.equals("backward")); this.edges = new AllowedEdges(edgesFmt); this.nodes = new LinkedHashSet(); this.paths = new ArrayList(); this.currentPath = new Stack(); this.visited = LongArrayBitVector.ofLength(graph.getNbNodes()); long nodeId = graph.getNodeId(src); if (output == OutputFmt.ONLY_NODES) { dfsOutputOnlyNodes(nodeId); } else { dfs(nodeId); } } public LinkedHashSet getNodes() { return nodes; } public ArrayList getPaths() { return paths; } private void dfs(long currentNodeId) { - nodes.add(graph.getSwhId(currentNodeId)); + SwhId currentSwhId = graph.getSwhId(currentNodeId); + nodes.add(currentSwhId); currentPath.push(currentNodeId); - long degree = graph.degree(currentNodeId, useTransposed); - LazyLongIterator neighbors = graph.neighbors(currentNodeId, useTransposed); long visitedNeighbors = 0; - - while (degree-- > 0) { - long neighborNodeId = neighbors.nextLong(); - Node.Type currentNodeType = graph.getSwhId(currentNodeId).getType(); - Node.Type neighborNodeType = graph.getSwhId(neighborNodeId).getType(); - if (edges.isAllowed(currentNodeType, neighborNodeType)) { + for (long neighborNodeId : new Neighbors(graph, useTransposed, edges, currentSwhId)) { dfs(neighborNodeId); visitedNeighbors++; - } } if (visitedNeighbors == 0) { SwhPath path = new SwhPath(); for (long nodeId : currentPath) { path.add(graph.getSwhId(nodeId)); } paths.add(path); } currentPath.pop(); } private void dfsOutputOnlyNodes(long currentNodeId) { - nodes.add(graph.getSwhId(currentNodeId)); + SwhId currentSwhId = graph.getSwhId(currentNodeId); + nodes.add(currentSwhId); visited.set(currentNodeId); - long degree = graph.degree(currentNodeId, useTransposed); - LazyLongIterator neighbors = graph.neighbors(currentNodeId, useTransposed); - - while (degree-- > 0) { - long neighborNodeId = neighbors.nextLong(); - Node.Type currentNodeType = graph.getSwhId(currentNodeId).getType(); - Node.Type neighborNodeType = graph.getSwhId(neighborNodeId).getType(); - if (!visited.getBoolean(neighborNodeId) - && edges.isAllowed(currentNodeType, neighborNodeType)) { + for (long neighborNodeId : new Neighbors(graph, useTransposed, edges, currentSwhId)) { + if (!visited.getBoolean(neighborNodeId)) { dfsOutputOnlyNodes(neighborNodeId); } } } } diff --git a/api/server/src/main/java/org/softwareheritage/graph/algo/Walk.java b/api/server/src/main/java/org/softwareheritage/graph/algo/Walk.java index 0467f9b..d6849f6 100644 --- a/api/server/src/main/java/org/softwareheritage/graph/algo/Walk.java +++ b/api/server/src/main/java/org/softwareheritage/graph/algo/Walk.java @@ -1,148 +1,136 @@ package org.softwareheritage.graph.algo; import java.util.Stack; import java.util.Queue; import java.util.LinkedList; -import it.unimi.dsi.big.webgraph.LazyLongIterator; import it.unimi.dsi.bits.LongArrayBitVector; import it.unimi.dsi.fastutil.longs.LongBigArrays; import org.softwareheritage.graph.AllowedEdges; import org.softwareheritage.graph.Graph; +import org.softwareheritage.graph.Neighbors; import org.softwareheritage.graph.Node; import org.softwareheritage.graph.SwhId; import org.softwareheritage.graph.SwhPath; public class Walk { Graph graph; boolean useTransposed; AllowedEdges edges; SwhPath path; long[][] nodeParent; public Walk( Graph graph, SwhId src, String dstFmt, String edgesFmt, String direction, String traversal) { if (!traversal.matches("dfs|bfs")) { throw new IllegalArgumentException("Unknown traversal algorithm: " + traversal); } if (!direction.matches("forward|backward")) { throw new IllegalArgumentException("Unknown traversal direction: " + direction); } this.graph = graph; this.useTransposed = (direction.equals("backward")); this.edges = new AllowedEdges(edgesFmt); this.path = new SwhPath(); this.nodeParent = LongBigArrays.newBigArray(graph.getNbNodes()); LongBigArrays.fill(nodeParent, -1); long srcNodeId = graph.getNodeId(src); long dstNodeId; if (traversal.equals("dfs")) { dstNodeId = dfs(srcNodeId, dstFmt); } else { dstNodeId = bfs(srcNodeId, dstFmt); } if (dstNodeId == -1) { throw new IllegalArgumentException("Unable to find destination point: " + dstFmt); } else { backtracking(srcNodeId, dstNodeId); } } public SwhPath getPath() { return path; } private void backtracking(long srcNodeId, long dstNodeId) { long currentNodeId = dstNodeId; while (currentNodeId != srcNodeId) { path.add(graph.getSwhId(currentNodeId)); currentNodeId = LongBigArrays.get(nodeParent, currentNodeId); } path.add(graph.getSwhId(srcNodeId)); path.reverse(); } private long dfs(long srcNodeId, String dstFmt) { Stack stack = new Stack(); LongArrayBitVector visited = LongArrayBitVector.ofLength(graph.getNbNodes()); stack.push(srcNodeId); visited.set(srcNodeId); while (!stack.isEmpty()) { long currentNodeId = stack.pop(); SwhId currentSwhId = graph.getSwhId(currentNodeId); if (isDestinationNode(currentSwhId, dstFmt)) { return currentNodeId; } - long degree = graph.degree(currentNodeId, useTransposed); - LazyLongIterator neighbors = graph.neighbors(currentNodeId, useTransposed); - - while (degree-- > 0) { - long neighborNodeId = neighbors.nextLong(); - SwhId neighborSwhId = graph.getSwhId(neighborNodeId); - if (!visited.getBoolean(neighborNodeId) - && edges.isAllowed(currentSwhId.getType(), neighborSwhId.getType())) { + for (long neighborNodeId : new Neighbors(graph, useTransposed, edges, currentSwhId)) { + if (!visited.getBoolean(neighborNodeId)) { stack.push(neighborNodeId); visited.set(neighborNodeId); LongBigArrays.set(nodeParent, neighborNodeId, currentNodeId); } } } return -1; } private long bfs(long srcNodeId, String dstFmt) { Queue queue = new LinkedList(); LongArrayBitVector visited = LongArrayBitVector.ofLength(graph.getNbNodes()); queue.add(srcNodeId); visited.set(srcNodeId); while (!queue.isEmpty()) { long currentNodeId = queue.poll(); SwhId currentSwhId = graph.getSwhId(currentNodeId); if (isDestinationNode(currentSwhId, dstFmt)) { return currentNodeId; } - long degree = graph.degree(currentNodeId, useTransposed); - LazyLongIterator neighbors = graph.neighbors(currentNodeId, useTransposed); - - while (degree-- > 0) { - long neighborNodeId = neighbors.nextLong(); - SwhId neighborSwhId = graph.getSwhId(neighborNodeId); - if (!visited.getBoolean(neighborNodeId) - && edges.isAllowed(currentSwhId.getType(), neighborSwhId.getType())) { + for (long neighborNodeId : new Neighbors(graph, useTransposed, edges, currentSwhId)) { + if (!visited.getBoolean(neighborNodeId)) { queue.add(neighborNodeId); visited.set(neighborNodeId); LongBigArrays.set(nodeParent, neighborNodeId, currentNodeId); } } } return -1; } private boolean isDestinationNode(SwhId swhId, String dstFmt) { // dstFmt is either a SwhId... if (swhId.toString().equals(dstFmt)) { return true; } // ...or a Node.Type try { Node.Type dstType = Node.Type.fromStr(dstFmt); return (swhId.getType().equals(dstType)); } catch (IllegalArgumentException e) { return false; } } }