diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ hooks: - id: codespell name: Check source code spelling - args: ["-L te,wth,alledges"] + args: ["-L te,wth,alledges,afterall"] stages: [commit] - repo: local @@ -48,3 +48,5 @@ args: ["-f", "java/pom.xml", "spotless:apply"] pass_filenames: false language: system + +exclude: ^swh/graph/rpc/ diff --git a/Makefile.local b/Makefile.local --- a/Makefile.local +++ b/Makefile.local @@ -9,6 +9,9 @@ java-%: mvn -f $(POM_PATH) $* +protoc: + python -m grpc_tools.protoc -I. --python_out=. --mypy_out=. --grpc_python_out=. swh/graph/rpc/*.proto + clean-java: java-clean .PHONY: java clean-java diff --git a/java/README.md b/java/README.md --- a/java/README.md +++ b/java/README.md @@ -15,13 +15,11 @@ ```bash $ java -cp target/swh-graph-*.jar \ - org.softwareheritage.graph.server.App \ + org.softwareheritage.graph.rpc.GraphServer \ ``` -Default port is 5009 (use the `--port` option to change port number). If you -need timings metadata send back to the client in addition to the result, use the -`--timings` flag. +Default port is 50091 (use the `--port` option to change port number). Tests ----- diff --git a/java/pom.xml b/java/pom.xml --- a/java/pom.xml +++ b/java/pom.xml @@ -14,6 +14,8 @@ UTF-8 11 + 3.20.1 + 1.46.0 @@ -28,43 +30,17 @@ 5.7.0 test - - org.junit.vintage - junit-vintage-engine - 5.7.0 - - - junit - junit - 4.12 - org.junit.jupiter junit-jupiter-engine 5.7.0 test - - org.hamcrest - hamcrest - 2.2 - test - - - io.javalin - javalin - 3.0.0 - org.slf4j slf4j-simple 1.7.26 - - com.fasterxml.jackson.core - jackson-databind - 2.13.0 - it.unimi.dsi webgraph-big @@ -117,11 +93,6 @@ jsap 2.1 - - net.sf.py4j - py4j - 0.10.9.3 - commons-codec commons-codec @@ -147,6 +118,41 @@ hadoop-client-runtime 3.3.1 + + com.google.protobuf + protobuf-java + ${protobuf.version} + + + io.grpc + grpc-netty-shaded + ${grpc.version} + + + io.grpc + grpc-protobuf + ${grpc.version} + + + io.grpc + grpc-stub + ${grpc.version} + + + io.grpc + grpc-services + ${grpc.version} + + + io.grpc + grpc-testing + ${grpc.version} + + + javax.annotation + javax.annotation-api + 1.3.2 + @@ -203,13 +209,17 @@ maven-project-info-reports-plugin 3.0.0 + + maven-dependency-plugin + 3.1.2 + maven-assembly-plugin 3.3.0 - org.softwareheritage.graph.server.App + org.softwareheritage.graph.rpc.GraphServer @@ -356,6 +366,33 @@ + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} + + + + + compile + compile-custom + test-compile + test-compile-custom + + + + + + + kr.motd.maven + os-maven-plugin + 1.6.2 + + diff --git a/java/src/main/java/org/softwareheritage/graph/Entry.java b/java/src/main/java/org/softwareheritage/graph/Entry.java deleted file mode 100644 --- a/java/src/main/java/org/softwareheritage/graph/Entry.java +++ /dev/null @@ -1,193 +0,0 @@ -package org.softwareheritage.graph; - -import java.io.*; -import java.util.ArrayList; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.PropertyNamingStrategy; - -public class Entry { - private SwhBidirectionalGraph graph; - - public void load_graph(String graphBasename) throws IOException { - System.err.println("Loading graph " + graphBasename + " ..."); - this.graph = SwhBidirectionalGraph.loadMapped(graphBasename); - System.err.println("Graph loaded."); - } - - public SwhBidirectionalGraph get_graph() { - return graph.copy(); - } - - public String stats() { - try { - Stats stats = new Stats(graph.getPath()); - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); - return objectMapper.writeValueAsString(stats); - } catch (IOException e) { - throw new RuntimeException("Cannot read stats: " + e); - } - } - - public void check_swhid(String src) { - graph.getNodeId(new SWHID(src)); - } - - private int count_visitor(NodeCountVisitor f, long srcNodeId) { - int[] count = {0}; - f.accept(srcNodeId, (node) -> { - count[0]++; - }); - return count[0]; - } - - public int count_leaves(String direction, String edgesFmt, String src, long maxEdges) { - long srcNodeId = graph.getNodeId(new SWHID(src)); - Traversal t = new Traversal(graph.copy(), direction, edgesFmt, maxEdges); - return count_visitor(t::leavesVisitor, srcNodeId); - } - - public int count_neighbors(String direction, String edgesFmt, String src, long maxEdges) { - long srcNodeId = graph.getNodeId(new SWHID(src)); - Traversal t = new Traversal(graph.copy(), direction, edgesFmt, maxEdges); - return count_visitor(t::neighborsVisitor, srcNodeId); - } - - public int count_visit_nodes(String direction, String edgesFmt, String src, long maxEdges) { - long srcNodeId = graph.getNodeId(new SWHID(src)); - Traversal t = new Traversal(graph.copy(), direction, edgesFmt, maxEdges); - return count_visitor(t::visitNodesVisitor, srcNodeId); - } - - public QueryHandler get_handler(String clientFIFO) { - return new QueryHandler(graph.copy(), clientFIFO); - } - - private interface NodeCountVisitor { - void accept(long nodeId, Traversal.NodeIdConsumer consumer); - } - - public class QueryHandler { - SwhBidirectionalGraph graph; - BufferedWriter out; - String clientFIFO; - - public QueryHandler(SwhBidirectionalGraph graph, String clientFIFO) { - this.graph = graph; - this.clientFIFO = clientFIFO; - this.out = null; - } - - public void writeNode(SWHID swhid) { - try { - out.write(swhid.toString() + "\n"); - } catch (IOException e) { - throw new RuntimeException("Cannot write response to client: " + e); - } - } - - public void writeEdge(SWHID src, SWHID dst) { - try { - out.write(src.toString() + " " + dst.toString() + "\n"); - } catch (IOException e) { - throw new RuntimeException("Cannot write response to client: " + e); - } - } - - public void open() { - try { - FileOutputStream file = new FileOutputStream(this.clientFIFO); - this.out = new BufferedWriter(new OutputStreamWriter(file)); - } catch (IOException e) { - throw new RuntimeException("Cannot open client FIFO: " + e); - } - } - - public void close() { - try { - out.close(); - } catch (IOException e) { - throw new RuntimeException("Cannot write response to client: " + e); - } - } - - public void leaves(String direction, String edgesFmt, String src, long maxEdges, String returnTypes) { - long srcNodeId = graph.getNodeId(new SWHID(src)); - open(); - Traversal t = new Traversal(graph, direction, edgesFmt, maxEdges, returnTypes); - for (Long nodeId : t.leaves(srcNodeId)) { - writeNode(graph.getSWHID(nodeId)); - } - close(); - } - - public void neighbors(String direction, String edgesFmt, String src, long maxEdges, String returnTypes) { - long srcNodeId = graph.getNodeId(new SWHID(src)); - open(); - Traversal t = new Traversal(graph, direction, edgesFmt, maxEdges, returnTypes); - for (Long nodeId : t.neighbors(srcNodeId)) { - writeNode(graph.getSWHID(nodeId)); - } - close(); - } - - public void visit_nodes(String direction, String edgesFmt, String src, long maxEdges, String returnTypes) { - long srcNodeId = graph.getNodeId(new SWHID(src)); - open(); - Traversal t = new Traversal(graph, direction, edgesFmt, maxEdges, returnTypes); - for (Long nodeId : t.visitNodes(srcNodeId)) { - writeNode(graph.getSWHID(nodeId)); - } - close(); - } - - public void visit_edges(String direction, String edgesFmt, String src, long maxEdges, String returnTypes) { - long srcNodeId = graph.getNodeId(new SWHID(src)); - open(); - Traversal t = new Traversal(graph, direction, edgesFmt, maxEdges); - t.visitNodesVisitor(srcNodeId, null, (srcId, dstId) -> { - writeEdge(graph.getSWHID(srcId), graph.getSWHID(dstId)); - }); - close(); - } - - public void walk(String direction, String edgesFmt, String algorithm, String src, String dst, long maxEdges, - String returnTypes) { - long srcNodeId = graph.getNodeId(new SWHID(src)); - open(); - ArrayList res; - Traversal t = new Traversal(graph, direction, edgesFmt, maxEdges, returnTypes); - if (dst.matches("ori|snp|rel|rev|dir|cnt")) { - Node.Type dstType = Node.Type.fromStr(dst); - res = t.walk(srcNodeId, dstType, algorithm); - } else { - long dstNodeId = graph.getNodeId(new SWHID(dst)); - res = t.walk(srcNodeId, dstNodeId, algorithm); - } - for (Long nodeId : res) { - writeNode(graph.getSWHID(nodeId)); - } - close(); - } - - public void random_walk(String direction, String edgesFmt, int retries, String src, String dst, long maxEdges, - String returnTypes) { - long srcNodeId = graph.getNodeId(new SWHID(src)); - open(); - ArrayList res; - Traversal t = new Traversal(graph, direction, edgesFmt, maxEdges, returnTypes); - if (dst.matches("ori|snp|rel|rev|dir|cnt")) { - Node.Type dstType = Node.Type.fromStr(dst); - res = t.randomWalk(srcNodeId, dstType, retries); - } else { - long dstNodeId = graph.getNodeId(new SWHID(dst)); - res = t.randomWalk(srcNodeId, dstNodeId, retries); - } - for (Long nodeId : res) { - writeNode(graph.getSWHID(nodeId)); - } - close(); - } - } -} diff --git a/java/src/main/java/org/softwareheritage/graph/Stats.java b/java/src/main/java/org/softwareheritage/graph/Stats.java deleted file mode 100644 --- a/java/src/main/java/org/softwareheritage/graph/Stats.java +++ /dev/null @@ -1,67 +0,0 @@ -package org.softwareheritage.graph; - -import java.io.FileInputStream; -import java.io.IOException; -import java.util.Properties; - -/** - * Statistics on the compressed graph. - *

- * These statistics are not computed but directly read from - * WebGraph generated .stats and .properties files. - * - * @author The Software Heritage developers - */ - -public class Stats { - public Counts counts; - public Ratios ratios; - public Degree indegree; - public Degree outdegree; - /** - * Constructor. - * - * @param graphPath path and basename of compressed graph - */ - public Stats(String graphPath) throws IOException { - Properties properties = new Properties(); - properties.load(new FileInputStream(graphPath + ".properties")); - properties.load(new FileInputStream(graphPath + ".stats")); - - this.counts = new Counts(); - this.ratios = new Ratios(); - this.indegree = new Degree(); - this.outdegree = new Degree(); - - this.counts.nodes = Long.parseLong(properties.getProperty("nodes")); - this.counts.edges = Long.parseLong(properties.getProperty("arcs")); - this.ratios.compression = Double.parseDouble(properties.getProperty("compratio")); - this.ratios.bitsPerNode = Double.parseDouble(properties.getProperty("bitspernode")); - this.ratios.bitsPerEdge = Double.parseDouble(properties.getProperty("bitsperlink")); - this.ratios.avgLocality = Double.parseDouble(properties.getProperty("avglocality")); - this.indegree.min = Long.parseLong(properties.getProperty("minindegree")); - this.indegree.max = Long.parseLong(properties.getProperty("maxindegree")); - this.indegree.avg = Double.parseDouble(properties.getProperty("avgindegree")); - this.outdegree.min = Long.parseLong(properties.getProperty("minoutdegree")); - this.outdegree.max = Long.parseLong(properties.getProperty("maxoutdegree")); - this.outdegree.avg = Double.parseDouble(properties.getProperty("avgoutdegree")); - } - - public static class Counts { - public long nodes; - public long edges; - } - - public static class Ratios { - public double compression; - public double bitsPerNode; - public double bitsPerEdge; - public double avgLocality; - } - - public static class Degree { - public long min; - public long max; - public double avg; - } -} diff --git a/java/src/main/java/org/softwareheritage/graph/SwhGraph.java b/java/src/main/java/org/softwareheritage/graph/SwhGraph.java --- a/java/src/main/java/org/softwareheritage/graph/SwhGraph.java +++ b/java/src/main/java/org/softwareheritage/graph/SwhGraph.java @@ -113,12 +113,12 @@ } /** @see SwhGraphProperties#getMessage(long) */ - default byte[] getMessage(long nodeId) throws IOException { + default byte[] getMessage(long nodeId) { return getProperties().getMessage(nodeId); } /** @see SwhGraphProperties#getUrl(long) */ - default String getUrl(long nodeId) throws IOException { + default String getUrl(long nodeId) { return getProperties().getUrl(nodeId); } @@ -128,7 +128,7 @@ } /** @see SwhGraphProperties#getTagName(long) */ - default byte[] getTagName(long nodeId) throws IOException { + default byte[] getTagName(long nodeId) { return getProperties().getTagName(nodeId); } diff --git a/java/src/main/java/org/softwareheritage/graph/SwhGraphProperties.java b/java/src/main/java/org/softwareheritage/graph/SwhGraphProperties.java --- a/java/src/main/java/org/softwareheritage/graph/SwhGraphProperties.java +++ b/java/src/main/java/org/softwareheritage/graph/SwhGraphProperties.java @@ -69,7 +69,6 @@ * Cleans up resources after use. */ public void close() throws IOException { - nodeIdMap.close(); edgeLabelNames.close(); } @@ -267,7 +266,7 @@ } /** Get the message of the given revision or release node */ - public byte[] getMessage(long nodeId) throws IOException { + public byte[] getMessage(long nodeId) { if (messageBuffer == null || messageOffsets == null) { throw new IllegalStateException("Messages not loaded"); } @@ -279,7 +278,7 @@ } /** Get the URL of the given origin node */ - public String getUrl(long nodeId) throws IOException { + public String getUrl(long nodeId) { byte[] url = getMessage(nodeId); return (url != null) ? new String(url) : null; } @@ -291,7 +290,7 @@ } /** Get the name of the given release node */ - public byte[] getTagName(long nodeId) throws IOException { + public byte[] getTagName(long nodeId) { if (tagNameBuffer == null || tagNameOffsets == null) { throw new IllegalStateException("Tag names not loaded"); } diff --git a/java/src/main/java/org/softwareheritage/graph/SwhPath.java b/java/src/main/java/org/softwareheritage/graph/SwhPath.java deleted file mode 100644 --- a/java/src/main/java/org/softwareheritage/graph/SwhPath.java +++ /dev/null @@ -1,122 +0,0 @@ -package org.softwareheritage.graph; - -import com.fasterxml.jackson.annotation.JsonValue; - -import java.util.ArrayList; - -/** - * Wrapper class to store a list of {@link SWHID}. - * - * @author The Software Heritage developers - * @see SWHID - */ - -public class SwhPath { - /** Internal list of {@link SWHID} */ - ArrayList path; - - /** - * Constructor. - */ - public SwhPath() { - this.path = new ArrayList<>(); - } - - /** - * Constructor. - * - * @param swhids variable number of string SWHIDs to initialize this path with - */ - public SwhPath(String... swhids) { - this(); - for (String swhid : swhids) { - add(new SWHID(swhid)); - } - } - - /** - * Constructor. - * - * @param swhids variable number of {@link SWHID} to initialize this path with - * @see SWHID - */ - public SwhPath(SWHID... swhids) { - this(); - for (SWHID swhid : swhids) { - add(swhid); - } - } - - /** - * Returns this path as a list of {@link SWHID}. - * - * @return list of {@link SWHID} constituting the path - * @see SWHID - */ - @JsonValue - public ArrayList getPath() { - return path; - } - - /** - * Adds a {@link SWHID} to this path. - * - * @param swhid {@link SWHID} to add to this path - * @see SWHID - */ - public void add(SWHID swhid) { - path.add(swhid); - } - - /** - * Returns the {@link SWHID} at the specified position in this path. - * - * @param index position of the {@link SWHID} to return - * @return {@link SWHID} at the specified position - * @see SWHID - */ - public SWHID get(int index) { - return path.get(index); - } - - /** - * Returns the number of elements in this path. - * - * @return number of elements in this path - */ - public int size() { - return path.size(); - } - - @Override - public boolean equals(Object otherObj) { - if (otherObj == this) - return true; - if (!(otherObj instanceof SwhPath)) - return false; - - SwhPath other = (SwhPath) otherObj; - if (size() != other.size()) { - return false; - } - - for (int i = 0; i < size(); i++) { - SWHID thisSWHID = get(i); - SWHID otherSWHID = other.get(i); - if (!thisSWHID.equals(otherSWHID)) { - return false; - } - } - - return true; - } - - @Override - public String toString() { - StringBuilder str = new StringBuilder(); - for (SWHID swhid : path) { - str.append(swhid).append("/"); - } - return str.toString(); - } -} diff --git a/java/src/main/java/org/softwareheritage/graph/SwhUnidirectionalGraph.java b/java/src/main/java/org/softwareheritage/graph/SwhUnidirectionalGraph.java --- a/java/src/main/java/org/softwareheritage/graph/SwhUnidirectionalGraph.java +++ b/java/src/main/java/org/softwareheritage/graph/SwhUnidirectionalGraph.java @@ -34,7 +34,7 @@ /** Property data of the graph (id/type mappings etc.) */ public SwhGraphProperties properties; - protected SwhUnidirectionalGraph(ImmutableGraph graph, SwhGraphProperties properties) { + public SwhUnidirectionalGraph(ImmutableGraph graph, SwhGraphProperties properties) { this.graph = graph; this.properties = properties; } diff --git a/java/src/main/java/org/softwareheritage/graph/Traversal.java b/java/src/main/java/org/softwareheritage/graph/Traversal.java --- a/java/src/main/java/org/softwareheritage/graph/Traversal.java +++ b/java/src/main/java/org/softwareheritage/graph/Traversal.java @@ -13,18 +13,13 @@ import java.util.function.Consumer; import java.util.function.LongConsumer; -import org.softwareheritage.graph.server.Endpoint; - import it.unimi.dsi.big.webgraph.LazyLongIterator; /** * Traversal algorithms on the compressed graph. *

- * Internal implementation of the traversal API endpoints. These methods only input/output internal - * long ids, which are converted in the {@link Endpoint} higher-level class to {@link SWHID}. * * @author The Software Heritage developers - * @see Endpoint */ public class Traversal { diff --git a/java/src/main/java/org/softwareheritage/graph/benchmark/AccessEdge.java b/java/src/main/java/org/softwareheritage/graph/benchmark/AccessEdge.java deleted file mode 100644 --- a/java/src/main/java/org/softwareheritage/graph/benchmark/AccessEdge.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.softwareheritage.graph.benchmark; - -import com.martiansoftware.jsap.JSAPException; -import it.unimi.dsi.big.webgraph.LazyLongIterator; -import org.softwareheritage.graph.SwhBidirectionalGraph; -import org.softwareheritage.graph.benchmark.utils.Statistics; -import org.softwareheritage.graph.benchmark.utils.Timing; - -import java.io.IOException; -import java.util.ArrayList; - -/** - * Benchmark to time edge access time. - * - * @author The Software Heritage developers - */ - -public class AccessEdge { - /** - * Main entrypoint. - * - * @param args command line arguments - */ - public static void main(String[] args) throws IOException, JSAPException { - Benchmark bench = new Benchmark(); - bench.parseCommandLineArgs(args); - - SwhBidirectionalGraph graph = SwhBidirectionalGraph.loadMapped(bench.args.graphPath); - - long[] nodeIds = bench.args.random.generateNodeIds(graph, bench.args.nbNodes); - - ArrayList timings = new ArrayList<>(); - for (long nodeId : nodeIds) { - long startTime = Timing.start(); - LazyLongIterator neighbors = graph.successors(nodeId); - long firstNeighbor = neighbors.nextLong(); - double duration = Timing.stop(startTime); - timings.add(duration); - } - - System.out.println("Used " + bench.args.nbNodes + " random edges (results are in seconds):"); - Statistics stats = new Statistics(timings); - stats.printAll(); - } -} diff --git a/java/src/main/java/org/softwareheritage/graph/benchmark/BFS.java b/java/src/main/java/org/softwareheritage/graph/benchmark/BFS.java deleted file mode 100644 --- a/java/src/main/java/org/softwareheritage/graph/benchmark/BFS.java +++ /dev/null @@ -1,107 +0,0 @@ -package org.softwareheritage.graph.benchmark; - -import com.google.common.primitives.Longs; -import com.martiansoftware.jsap.*; -import it.unimi.dsi.big.webgraph.ImmutableGraph; -import it.unimi.dsi.big.webgraph.LazyLongIterator; -import it.unimi.dsi.bits.LongArrayBitVector; -import it.unimi.dsi.fastutil.Arrays; -import it.unimi.dsi.io.ByteDiskQueue; -import it.unimi.dsi.logging.ProgressLogger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.softwareheritage.graph.SwhBidirectionalGraph; - -import java.io.File; -import java.io.IOException; - -public class BFS { - private final static Logger LOGGER = LoggerFactory.getLogger(BFS.class); - private final ImmutableGraph graph; - - public BFS(ImmutableGraph graph) { - this.graph = graph; - } - - private static JSAPResult parse_args(String[] args) { - JSAPResult config = null; - try { - SimpleJSAP jsap = new SimpleJSAP(BFS.class.getName(), "", - new Parameter[]{ - new FlaggedOption("graphPath", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, 'g', - "graph", "Basename of the compressed graph"), - - new FlaggedOption("useTransposed", JSAP.BOOLEAN_PARSER, "false", JSAP.NOT_REQUIRED, 'T', - "transposed", "Use transposed graph (default: false)"),}); - - config = jsap.parse(args); - if (jsap.messagePrinted()) { - System.exit(1); - } - } catch (JSAPException e) { - e.printStackTrace(); - } - return config; - } - - public static void main(String[] args) throws IOException { - JSAPResult config = parse_args(args); - String graphPath = config.getString("graphPath"); - boolean useTransposed = config.getBoolean("useTransposed"); - - System.err.println("Loading graph " + graphPath + " ..."); - SwhBidirectionalGraph graph = SwhBidirectionalGraph.loadMapped(graphPath); - System.err.println("Graph loaded."); - - if (useTransposed) - graph = graph.transpose(); - - BFS bfs = new BFS(graph); - bfs.bfsperm(); - } - - // Partly inlined from it.unimi.dsi.law.big.graph.BFS - private void bfsperm() throws IOException { - final long n = graph.numNodes(); - // Allow enough memory to behave like in-memory queue - int bufferSize = (int) Math.min(Arrays.MAX_ARRAY_SIZE & ~0x7, 8L * n); - - // Use a disk based queue to store BFS frontier - final File queueFile = File.createTempFile(BFS.class.getSimpleName(), "queue"); - final ByteDiskQueue queue = ByteDiskQueue.createNew(queueFile, bufferSize, true); - final byte[] byteBuf = new byte[Long.BYTES]; - // WARNING: no 64-bit version of this data-structure, but it can support - // indices up to 2^37 - final LongArrayBitVector visited = LongArrayBitVector.ofLength(n); - final ProgressLogger pl = new ProgressLogger(LOGGER); - pl.expectedUpdates = n; - pl.itemsName = "nodes"; - pl.start("Starting breadth-first visit..."); - - for (long i = 0; i < n; i++) { - if (visited.getBoolean(i)) - continue; - queue.enqueue(Longs.toByteArray(i)); - visited.set(i); - - while (!queue.isEmpty()) { - queue.dequeue(byteBuf); - final long currentNode = Longs.fromByteArray(byteBuf); - - final LazyLongIterator iterator = graph.successors(currentNode); - long succ; - while ((succ = iterator.nextLong()) != -1) { - if (!visited.getBoolean(succ)) { - visited.set(succ); - queue.enqueue(Longs.toByteArray(succ)); - } - } - - pl.update(); - } - } - - pl.done(); - queue.close(); - } -} diff --git a/java/src/main/java/org/softwareheritage/graph/benchmark/Benchmark.java b/java/src/main/java/org/softwareheritage/graph/benchmark/Benchmark.java deleted file mode 100644 --- a/java/src/main/java/org/softwareheritage/graph/benchmark/Benchmark.java +++ /dev/null @@ -1,154 +0,0 @@ -package org.softwareheritage.graph.benchmark; - -import com.martiansoftware.jsap.*; -import org.softwareheritage.graph.SwhBidirectionalGraph; -import org.softwareheritage.graph.SWHID; -import org.softwareheritage.graph.benchmark.utils.Random; -import org.softwareheritage.graph.benchmark.utils.Statistics; -import org.softwareheritage.graph.server.Endpoint; - -import java.io.BufferedWriter; -import java.io.FileWriter; -import java.io.IOException; -import java.io.Writer; -import java.util.ArrayList; -import java.util.StringJoiner; -import java.util.function.Function; - -/** - * Benchmark common utility functions. - * - * @author The Software Heritage developers - */ - -public class Benchmark { - /** CSV separator for log file */ - final String CSV_SEPARATOR = ";"; - /** Command line arguments */ - public Args args; - /** - * Constructor. - */ - public Benchmark() { - this.args = new Args(); - } - - /** - * Parses benchmark command line arguments. - * - * @param args command line arguments - */ - public void parseCommandLineArgs(String[] args) throws JSAPException { - SimpleJSAP jsap = new SimpleJSAP(Benchmark.class.getName(), - "Benchmark tool for Software Heritage use-cases scenarios.", - new Parameter[]{ - new UnflaggedOption("graphPath", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, - JSAP.NOT_GREEDY, "The basename of the compressed graph."), - new FlaggedOption("nbNodes", JSAP.INTEGER_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, 'n', - "nb-nodes", "Number of random nodes used to do the benchmark."), - new FlaggedOption("logFile", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, 'l', - "log-file", "File name to output CSV format benchmark log."), - new FlaggedOption("seed", JSAP.LONG_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 's', "seed", - "Random generator seed."),}); - - JSAPResult config = jsap.parse(args); - if (jsap.messagePrinted()) { - System.exit(1); - } - - this.args.graphPath = config.getString("graphPath"); - this.args.nbNodes = config.getInt("nbNodes"); - this.args.logFile = config.getString("logFile"); - this.args.random = config.contains("seed") ? new Random(config.getLong("seed")) : new Random(); - } - - /** - * Creates CSV file for log output. - */ - public void createCSVLogFile() throws IOException { - try (Writer csvLog = new BufferedWriter(new FileWriter(args.logFile))) { - StringJoiner csvHeader = new StringJoiner(CSV_SEPARATOR); - csvHeader.add("use case name").add("SWHID").add("number of edges accessed").add("traversal timing") - .add("swhid2node timing").add("node2swhid timing"); - csvLog.write(csvHeader.toString() + "\n"); - } - } - - /** - * Times a specific endpoint and outputs individual datapoints along with aggregated statistics. - * - * @param useCaseName benchmark use-case name - * @param graph compressed graph used in the benchmark - * @param nodeIds node ids to use as starting point for the endpoint traversal - * @param operation endpoint function to benchmark - * @param dstFmt destination formatted string as described in the - * API - * @param algorithm traversal algorithm used in endpoint call (either "dfs" or "bfs") - */ - public void timeEndpoint(String useCaseName, SwhBidirectionalGraph graph, long[] nodeIds, - Function operation, String dstFmt, String algorithm) throws IOException { - ArrayList timings = new ArrayList<>(); - ArrayList timingsNormalized = new ArrayList<>(); - ArrayList nbEdgesAccessed = new ArrayList<>(); - - final boolean append = true; - try (Writer csvLog = new BufferedWriter(new FileWriter(args.logFile, append))) { - for (long nodeId : nodeIds) { - SWHID swhid = graph.getSWHID(nodeId); - - Endpoint.Output output = (dstFmt == null) - ? operation.apply(new Endpoint.Input(swhid)) - : operation.apply(new Endpoint.Input(swhid, dstFmt, algorithm)); - - StringJoiner csvLine = new StringJoiner(CSV_SEPARATOR); - csvLine.add(useCaseName).add(swhid.toString()).add(Long.toString(output.meta.nbEdgesAccessed)) - .add(Double.toString(output.meta.timings.traversal)) - .add(Double.toString(output.meta.timings.swhid2node)) - .add(Double.toString(output.meta.timings.node2swhid)); - csvLog.write(csvLine.toString() + "\n"); - - timings.add(output.meta.timings.traversal); - nbEdgesAccessed.add((double) output.meta.nbEdgesAccessed); - if (output.meta.nbEdgesAccessed != 0) { - timingsNormalized.add(output.meta.timings.traversal / output.meta.nbEdgesAccessed); - } - } - } - - System.out.println("\n" + useCaseName + " use-case:"); - - System.out.println("timings:"); - Statistics stats = new Statistics(timings); - stats.printAll(); - - System.out.println("timings normalized:"); - Statistics statsNormalized = new Statistics(timingsNormalized); - statsNormalized.printAll(); - - System.out.println("nb edges accessed:"); - Statistics statsNbEdgesAccessed = new Statistics(nbEdgesAccessed); - statsNbEdgesAccessed.printAll(); - } - - /** - * Same as {@link #timeEndpoint} but without destination or algorithm specified to endpoint call. - */ - public void timeEndpoint(String useCaseName, SwhBidirectionalGraph graph, long[] nodeIds, - Function operation) throws IOException { - timeEndpoint(useCaseName, graph, nodeIds, operation, null, null); - } - - /** - * Input arguments. - */ - public class Args { - /** Basename of the compressed graph */ - public String graphPath; - /** Number of random nodes to use for the benchmark */ - public int nbNodes; - /** File name for CSV format benchmark log */ - public String logFile; - /** Random generator */ - public Random random; - } -} diff --git a/java/src/main/java/org/softwareheritage/graph/benchmark/Browsing.java b/java/src/main/java/org/softwareheritage/graph/benchmark/Browsing.java deleted file mode 100644 --- a/java/src/main/java/org/softwareheritage/graph/benchmark/Browsing.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.softwareheritage.graph.benchmark; - -import com.martiansoftware.jsap.JSAPException; -import org.softwareheritage.graph.SwhBidirectionalGraph; -import org.softwareheritage.graph.Node; -import org.softwareheritage.graph.server.Endpoint; - -import java.io.IOException; - -/** - * Benchmark Software Heritage - * browsing - * use-cases scenarios. - * - * @author The Software Heritage developers - */ - -public class Browsing { - /** - * Main entrypoint. - * - * @param args command line arguments - */ - public static void main(String[] args) throws IOException, JSAPException { - Benchmark bench = new Benchmark(); - bench.parseCommandLineArgs(args); - - SwhBidirectionalGraph graph = SwhBidirectionalGraph.loadMapped(bench.args.graphPath); - - long[] dirNodeIds = bench.args.random.generateNodeIdsOfType(graph, bench.args.nbNodes, Node.Type.DIR); - long[] revNodeIds = bench.args.random.generateNodeIdsOfType(graph, bench.args.nbNodes, Node.Type.REV); - - Endpoint dirEndpoint = new Endpoint(graph, "forward", "dir:cnt,dir:dir"); - Endpoint revEndpoint = new Endpoint(graph, "forward", "rev:rev"); - - System.out.println("Used " + bench.args.nbNodes + " random nodes (results are in seconds):"); - bench.createCSVLogFile(); - bench.timeEndpoint("ls", graph, dirNodeIds, dirEndpoint::neighbors); - bench.timeEndpoint("ls -R", graph, dirNodeIds, dirEndpoint::visitPaths); - bench.timeEndpoint("git log", graph, revNodeIds, revEndpoint::visitNodes); - } -} diff --git a/java/src/main/java/org/softwareheritage/graph/benchmark/Provenance.java b/java/src/main/java/org/softwareheritage/graph/benchmark/Provenance.java deleted file mode 100644 --- a/java/src/main/java/org/softwareheritage/graph/benchmark/Provenance.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.softwareheritage.graph.benchmark; - -import com.martiansoftware.jsap.JSAPException; -import org.softwareheritage.graph.SwhBidirectionalGraph; -import org.softwareheritage.graph.server.Endpoint; - -import java.io.IOException; - -/** - * Benchmark Software Heritage - * provenance - * use-cases scenarios. - * - * @author The Software Heritage developers - */ - -public class Provenance { - /** - * Main entrypoint. - * - * @param args command line arguments - */ - public static void main(String[] args) throws IOException, JSAPException { - Benchmark bench = new Benchmark(); - bench.parseCommandLineArgs(args); - - SwhBidirectionalGraph graph = SwhBidirectionalGraph.loadMapped(bench.args.graphPath); - - long[] nodeIds = bench.args.random.generateNodeIds(graph, bench.args.nbNodes); - - Endpoint commitProvenanceEndpoint = new Endpoint(graph, "backward", "dir:dir,cnt:dir,dir:rev"); - Endpoint originProvenanceEndpoint = new Endpoint(graph, "backward", "*"); - - System.out.println("Used " + bench.args.nbNodes + " random nodes (results are in seconds):"); - bench.createCSVLogFile(); - - bench.timeEndpoint("commit provenance (dfs)", graph, nodeIds, commitProvenanceEndpoint::walk, "rev", "dfs"); - bench.timeEndpoint("commit provenance (bfs)", graph, nodeIds, commitProvenanceEndpoint::walk, "rev", "bfs"); - bench.timeEndpoint("complete commit provenance", graph, nodeIds, commitProvenanceEndpoint::leaves); - - bench.timeEndpoint("origin provenance (dfs)", graph, nodeIds, originProvenanceEndpoint::walk, "ori", "dfs"); - bench.timeEndpoint("origin provenance (bfs)", graph, nodeIds, originProvenanceEndpoint::walk, "ori", "bfs"); - bench.timeEndpoint("complete origin provenance", graph, nodeIds, originProvenanceEndpoint::leaves); - } -} diff --git a/java/src/main/java/org/softwareheritage/graph/benchmark/Vault.java b/java/src/main/java/org/softwareheritage/graph/benchmark/Vault.java deleted file mode 100644 --- a/java/src/main/java/org/softwareheritage/graph/benchmark/Vault.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.softwareheritage.graph.benchmark; - -import com.martiansoftware.jsap.JSAPException; -import org.softwareheritage.graph.SwhBidirectionalGraph; -import org.softwareheritage.graph.server.Endpoint; - -import java.io.IOException; - -/** - * Benchmark Software Heritage - * vault use-case - * scenario. - * - * @author The Software Heritage developers - */ - -public class Vault { - /** - * Main entrypoint. - * - * @param args command line arguments - */ - public static void main(String[] args) throws IOException, JSAPException { - Benchmark bench = new Benchmark(); - bench.parseCommandLineArgs(args); - - SwhBidirectionalGraph graph = SwhBidirectionalGraph.loadMapped(bench.args.graphPath); - - long[] nodeIds = bench.args.random.generateNodeIds(graph, bench.args.nbNodes); - - Endpoint endpoint = new Endpoint(graph, "forward", "*"); - - System.out.println("Used " + bench.args.nbNodes + " random nodes (results are in seconds):"); - bench.createCSVLogFile(); - bench.timeEndpoint("git bundle", graph, nodeIds, endpoint::visitNodes); - } -} diff --git a/java/src/main/java/org/softwareheritage/graph/benchmark/utils/Random.java b/java/src/main/java/org/softwareheritage/graph/benchmark/utils/Random.java deleted file mode 100644 --- a/java/src/main/java/org/softwareheritage/graph/benchmark/utils/Random.java +++ /dev/null @@ -1,67 +0,0 @@ -package org.softwareheritage.graph.benchmark.utils; - -import org.softwareheritage.graph.SwhBidirectionalGraph; -import org.softwareheritage.graph.Node; - -import java.util.PrimitiveIterator; - -/** - * Random related utility class. - * - * @author The Software Heritage developers - */ - -public class Random { - /** Internal pseudorandom generator */ - java.util.Random random; - - /** - * Constructor. - */ - public Random() { - this.random = new java.util.Random(); - } - - /** - * Constructor. - * - * @param seed random generator seed - */ - public Random(long seed) { - this.random = new java.util.Random(seed); - } - - /** - * Generates random node ids. - * - * @param graph graph used to pick node ids - * @param nbNodes number of node ids to generate - * @return an array of random node ids - */ - public long[] generateNodeIds(SwhBidirectionalGraph graph, int nbNodes) { - return random.longs(nbNodes, 0, graph.numNodes()).toArray(); - } - - /** - * Generates random node ids with a specific type. - * - * @param graph graph used to pick node ids - * @param nbNodes number of node ids to generate - * @param expectedType specific node type to pick - * @return an array of random node ids - */ - public long[] generateNodeIdsOfType(SwhBidirectionalGraph graph, int nbNodes, Node.Type expectedType) { - PrimitiveIterator.OfLong nodes = random.longs(0, graph.numNodes()).iterator(); - long[] nodeIds = new long[nbNodes]; - - long nextId; - for (int i = 0; i < nbNodes; i++) { - do { - nextId = nodes.nextLong(); - } while (graph.getNodeType(nextId) != expectedType); - nodeIds[i] = nextId; - } - - return nodeIds; - } -} diff --git a/java/src/main/java/org/softwareheritage/graph/benchmark/utils/Statistics.java b/java/src/main/java/org/softwareheritage/graph/benchmark/utils/Statistics.java deleted file mode 100644 --- a/java/src/main/java/org/softwareheritage/graph/benchmark/utils/Statistics.java +++ /dev/null @@ -1,104 +0,0 @@ -package org.softwareheritage.graph.benchmark.utils; - -import java.util.ArrayList; -import java.util.Collections; - -/** - * Compute various statistics on a list of values. - * - * @author The Software Heritage developers - */ - -public class Statistics { - /** Input values */ - ArrayList values; - - /** - * Constructor. - * - * @param values input values - */ - public Statistics(ArrayList values) { - this.values = values; - } - - /** - * Returns the minimum value. - * - * @return minimum value - */ - public double getMin() { - double min = Double.POSITIVE_INFINITY; - for (double v : values) { - min = Math.min(min, v); - } - return min; - } - - /** - * Returns the maximum value. - * - * @return maximum value - */ - public double getMax() { - double max = Double.NEGATIVE_INFINITY; - for (double v : values) { - max = Math.max(max, v); - } - return max; - } - - /** - * Computes the average. - * - * @return average value - */ - public double getAverage() { - double sum = 0; - for (double v : values) { - sum += v; - } - return sum / (double) values.size(); - } - - /** - * Returns the median value. - * - * @return median value - */ - public double getMedian() { - Collections.sort(values); - int length = values.size(); - if (length % 2 == 0) { - return (values.get(length / 2) + values.get(length / 2 - 1)) / 2; - } else { - return values.get(length / 2); - } - } - - /** - * Computes the standard deviation. - * - * @return standard deviation value - */ - public double getStandardDeviation() { - double average = getAverage(); - double variance = 0; - for (double v : values) { - variance += (v - average) * (v - average); - } - variance /= (double) values.size(); - return Math.sqrt(variance); - } - - /** - * Computes and prints all statistical values. - */ - public void printAll() { - System.out.println("min value: " + getMin()); - System.out.println("max value: " + getMax()); - System.out.println("average: " + getAverage()); - System.out.println("median: " + getMedian()); - System.out.println("standard deviation: " + getStandardDeviation()); - } -} diff --git a/java/src/main/java/org/softwareheritage/graph/benchmark/utils/Timing.java b/java/src/main/java/org/softwareheritage/graph/benchmark/utils/Timing.java deleted file mode 100644 --- a/java/src/main/java/org/softwareheritage/graph/benchmark/utils/Timing.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.softwareheritage.graph.benchmark.utils; - -/** - * Time measurement utility class. - * - * @author The Software Heritage developers - */ - -public class Timing { - /** - * Returns measurement starting timestamp. - * - * @return timestamp used for time measurement - */ - public static long start() { - return System.nanoTime(); - } - - /** - * Ends timing measurement and returns total duration in seconds. - * - * @param startTime measurement starting timestamp - * @return time in seconds elapsed since starting point - */ - public static double stop(long startTime) { - long endTime = System.nanoTime(); - double duration = (double) (endTime - startTime) / 1_000_000_000; - return duration; - } -} diff --git a/java/src/main/java/org/softwareheritage/graph/experiments/multiplicationfactor/GenDistribution.java b/java/src/main/java/org/softwareheritage/graph/experiments/multiplicationfactor/GenDistribution.java --- a/java/src/main/java/org/softwareheritage/graph/experiments/multiplicationfactor/GenDistribution.java +++ b/java/src/main/java/org/softwareheritage/graph/experiments/multiplicationfactor/GenDistribution.java @@ -4,7 +4,6 @@ import org.softwareheritage.graph.SwhBidirectionalGraph; import org.softwareheritage.graph.Node; import org.softwareheritage.graph.Traversal; -import org.softwareheritage.graph.benchmark.utils.Timing; import java.io.IOException; import java.util.Scanner; @@ -106,13 +105,14 @@ Traversal t = new Traversal(thread_graph, "backward", edgesFmt); int[] count = {0}; - startTime = Timing.start(); + startTime = System.nanoTime(); t.visitNodesVisitor(node, (curnode) -> { if (tp.graph.getNodeType(curnode) == dstType) { count[0]++; } }); - totalTime = Timing.stop(startTime); + long endTime = System.nanoTime(); + totalTime = (double) (endTime - startTime) / 1e9; System.out.format("%d %d %d %d %f\n", node, count[0], t.getNbNodesAccessed(), t.getNbEdgesAccessed(), totalTime); } diff --git a/java/src/main/java/org/softwareheritage/graph/maps/MapFile.java b/java/src/main/java/org/softwareheritage/graph/maps/MapFile.java deleted file mode 100644 --- a/java/src/main/java/org/softwareheritage/graph/maps/MapFile.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.softwareheritage.graph.maps; - -import it.unimi.dsi.io.ByteBufferInputStream; - -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.channels.FileChannel; - -/** - * Wrapper class around very big mmap()-ed file. - *

- * Java has a limit for mmap()-ed files because of unsupported 64-bit indexing. The - * dsiutils ByteBufferInputStream is used to overcome - * this Java limit. - * - * @author The Software Heritage developers - */ - -public class MapFile { - /** Memory-mapped file buffer */ - ByteBufferInputStream bufferMap; - /** Fixed line length of the mmap()-ed file */ - int lineLength; - - /** - * Constructor. - * - * @param path file path to mmap() - * @param lineLength fixed length of a line in the file - */ - public MapFile(String path, int lineLength) throws IOException { - this.bufferMap = null; - this.lineLength = lineLength; - - try (RandomAccessFile mapFile = new RandomAccessFile(new File(path), "r")) { - FileChannel fileChannel = mapFile.getChannel(); - bufferMap = ByteBufferInputStream.map(fileChannel, FileChannel.MapMode.READ_ONLY); - } - } - - /** - * Returns a specific line in the file. - * - * @param lineIndex line number in the file - * @return the line at the specified position - */ - public byte[] readAtLine(long lineIndex) { - byte[] buffer = new byte[lineLength]; - long position = lineIndex * (long) lineLength; - bufferMap.position(position); - bufferMap.read(buffer, 0, lineLength); - return buffer; - } - - public long size() { - return bufferMap.length() / (long) lineLength; - } - - /** - * Closes the mmap()-ed file. - */ - public void close() throws IOException { - bufferMap.close(); - } -} diff --git a/java/src/main/java/org/softwareheritage/graph/maps/NodeIdMap.java b/java/src/main/java/org/softwareheritage/graph/maps/NodeIdMap.java --- a/java/src/main/java/org/softwareheritage/graph/maps/NodeIdMap.java +++ b/java/src/main/java/org/softwareheritage/graph/maps/NodeIdMap.java @@ -1,17 +1,18 @@ package org.softwareheritage.graph.maps; import it.unimi.dsi.fastutil.Size64; +import it.unimi.dsi.fastutil.bytes.ByteBigList; +import it.unimi.dsi.fastutil.bytes.ByteMappedBigList; import it.unimi.dsi.fastutil.io.BinIO; import it.unimi.dsi.fastutil.longs.LongBigList; +import it.unimi.dsi.fastutil.longs.LongMappedBigList; import it.unimi.dsi.fastutil.objects.Object2LongFunction; -import it.unimi.dsi.util.ByteBufferLongBigList; import org.softwareheritage.graph.SWHID; import org.softwareheritage.graph.compress.NodeMapBuilder; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; -import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; /** @@ -38,7 +39,7 @@ String graphPath; /** mmap()-ed NODE_TO_SWHID file */ - MapFile nodeToSwhMap; + ByteBigList nodeToSwhMap; /** Minimal perfect hash (MPH) function SWHID -> initial order */ Object2LongFunction mph; @@ -54,14 +55,14 @@ this.graphPath = graphPath; // node -> SWHID - this.nodeToSwhMap = new MapFile(graphPath + NODE_TO_SWHID, SWHID_BIN_SIZE); + try (RandomAccessFile raf = new RandomAccessFile(graphPath + NODE_TO_SWHID, "r")) { + this.nodeToSwhMap = ByteMappedBigList.map(raf.getChannel()); + } // SWHID -> node this.mph = loadMph(graphPath + ".mph"); - try (RandomAccessFile mapFile = new RandomAccessFile(new File(graphPath + ".order"), "r")) { - FileChannel fileChannel = mapFile.getChannel(); - this.orderMap = ByteBufferLongBigList.map(fileChannel); + this.orderMap = LongMappedBigList.map(mapFile.getChannel()); } } @@ -95,6 +96,7 @@ return legacyFunction.getLong(new String(bi, StandardCharsets.UTF_8)); } + @SuppressWarnings("deprecation") @Override public int size() { return legacyFunction.size(); @@ -169,23 +171,19 @@ * Each line in NODE_TO_SWHID is formatted as: swhid The file is ordered by nodeId, meaning node0's * swhid is at line 0, hence we can read the nodeId-th line to get corresponding swhid */ - if (nodeId < 0 || nodeId >= nodeToSwhMap.size()) { - throw new IllegalArgumentException("Node id " + nodeId + " should be between 0 and " + nodeToSwhMap.size()); + if (nodeId < 0 || nodeId >= nodeToSwhMap.size64()) { + throw new IllegalArgumentException( + "Node id " + nodeId + " should be between 0 and " + nodeToSwhMap.size64()); } - return SWHID.fromBytes(nodeToSwhMap.readAtLine(nodeId)); - } - - /** - * Closes the mapping files. - */ - public void close() throws IOException { - nodeToSwhMap.close(); + byte[] swhid = new byte[SWHID_BIN_SIZE]; + nodeToSwhMap.getElements(nodeId * SWHID_BIN_SIZE, swhid, 0, SWHID_BIN_SIZE); + return SWHID.fromBytes(swhid); } /** Return the number of nodes in the map. */ @Override public long size64() { - return nodeToSwhMap.size(); + return nodeToSwhMap.size64(); } } diff --git a/java/src/main/java/org/softwareheritage/graph/rpc/GraphServer.java b/java/src/main/java/org/softwareheritage/graph/rpc/GraphServer.java new file mode 100644 --- /dev/null +++ b/java/src/main/java/org/softwareheritage/graph/rpc/GraphServer.java @@ -0,0 +1,203 @@ +package org.softwareheritage.graph.rpc; + +import com.martiansoftware.jsap.*; +import io.grpc.Server; +import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; +import io.grpc.netty.shaded.io.netty.channel.ChannelOption; +import io.grpc.stub.StreamObserver; +import io.grpc.protobuf.services.ProtoReflectionService; +import it.unimi.dsi.logging.ProgressLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.softwareheritage.graph.SWHID; +import org.softwareheritage.graph.SwhBidirectionalGraph; +import org.softwareheritage.graph.compress.LabelMapBuilder; + +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Properties; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Server that manages startup/shutdown of a {@code Greeter} server. + */ +public class GraphServer { + private final static Logger logger = LoggerFactory.getLogger(GraphServer.class); + + private final SwhBidirectionalGraph graph; + private final int port; + private final int threads; + private Server server; + + public GraphServer(String graphBasename, int port, int threads) throws IOException { + // TODO: use loadLabelledMapped() when https://github.com/vigna/webgraph-big/pull/5 is merged + this.graph = SwhBidirectionalGraph.loadLabelled(graphBasename, new ProgressLogger(logger)); + this.port = port; + this.threads = threads; + graph.loadContentLength(); + graph.loadContentIsSkipped(); + graph.loadPersonIds(); + graph.loadAuthorTimestamps(); + graph.loadCommitterTimestamps(); + graph.loadMessages(); + graph.loadTagNames(); + graph.loadLabelNames(); + } + + private void start() throws IOException { + server = NettyServerBuilder.forPort(port).withChildOption(ChannelOption.SO_REUSEADDR, true) + .executor(Executors.newFixedThreadPool(threads)).addService(new TraversalService(graph)) + .addService(ProtoReflectionService.newInstance()).build().start(); + logger.info("Server started, listening on " + port); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + GraphServer.this.stop(); + } catch (InterruptedException e) { + e.printStackTrace(System.err); + } + })); + } + + private void stop() throws InterruptedException { + if (server != null) { + server.shutdown().awaitTermination(30, TimeUnit.SECONDS); + } + } + + /** + * Await termination on the main thread since the grpc library uses daemon threads. + */ + private void blockUntilShutdown() throws InterruptedException { + if (server != null) { + server.awaitTermination(); + } + } + + private static JSAPResult parseArgs(String[] args) { + JSAPResult config = null; + try { + SimpleJSAP jsap = new SimpleJSAP(LabelMapBuilder.class.getName(), "", + new Parameter[]{ + new FlaggedOption("port", JSAP.INTEGER_PARSER, "50091", JSAP.NOT_REQUIRED, 'p', "port", + "The port on which the server should listen."), + new FlaggedOption("threads", JSAP.INTEGER_PARSER, "1", JSAP.NOT_REQUIRED, 't', "threads", + "The number of concurrent threads. 0 = number of cores."), + new UnflaggedOption("graphBasename", JSAP.STRING_PARSER, JSAP.REQUIRED, + "Basename of the output graph")}); + + config = jsap.parse(args); + if (jsap.messagePrinted()) { + System.exit(1); + } + } catch (JSAPException e) { + e.printStackTrace(); + } + return config; + } + + /** + * Main launches the server from the command line. + */ + public static void main(String[] args) throws IOException, InterruptedException { + JSAPResult config = parseArgs(args); + String graphBasename = config.getString("graphBasename"); + int port = config.getInt("port"); + int threads = config.getInt("threads"); + if (threads == 0) { + threads = Runtime.getRuntime().availableProcessors(); + } + + final GraphServer server = new GraphServer(graphBasename, port, threads); + server.start(); + server.blockUntilShutdown(); + } + + static class TraversalService extends TraversalServiceGrpc.TraversalServiceImplBase { + SwhBidirectionalGraph graph; + + public TraversalService(SwhBidirectionalGraph graph) { + this.graph = graph; + } + + @Override + public void checkSwhid(CheckSwhidRequest request, StreamObserver responseObserver) { + boolean exists = true; + CheckSwhidResponse.Builder builder = CheckSwhidResponse.newBuilder().setExists(true); + try { + graph.getNodeId(new SWHID(request.getSwhid())); + } catch (IllegalArgumentException e) { + builder.setExists(false); + builder.setDetails(e.getMessage()); + } + responseObserver.onNext(builder.build()); + responseObserver.onCompleted(); + } + + @Override + public void stats(StatsRequest request, StreamObserver responseObserver) { + StatsResponse.Builder response = StatsResponse.newBuilder(); + response.setNumNodes(graph.numNodes()); + response.setNumEdges(graph.numArcs()); + + Properties properties = new Properties(); + try { + properties.load(new FileInputStream(graph.getPath() + ".properties")); + properties.load(new FileInputStream(graph.getPath() + ".stats")); + } catch (IOException e) { + throw new RuntimeException(e); + } + response.setCompression(Double.parseDouble(properties.getProperty("compratio"))); + response.setBitsPerNode(Double.parseDouble(properties.getProperty("bitspernode"))); + response.setBitsPerEdge(Double.parseDouble(properties.getProperty("bitsperlink"))); + response.setAvgLocality(Double.parseDouble(properties.getProperty("avglocality"))); + response.setIndegreeMin(Long.parseLong(properties.getProperty("minindegree"))); + response.setIndegreeMax(Long.parseLong(properties.getProperty("maxindegree"))); + response.setIndegreeAvg(Double.parseDouble(properties.getProperty("avgindegree"))); + response.setOutdegreeMin(Long.parseLong(properties.getProperty("minoutdegree"))); + response.setOutdegreeMax(Long.parseLong(properties.getProperty("maxoutdegree"))); + response.setOutdegreeAvg(Double.parseDouble(properties.getProperty("avgoutdegree"))); + responseObserver.onNext(response.build()); + responseObserver.onCompleted(); + } + + @Override + public void traverse(TraversalRequest request, StreamObserver responseObserver) { + SwhBidirectionalGraph g = graph.copy(); + Traversal.simpleTraversal(g, request, responseObserver::onNext); + responseObserver.onCompleted(); + } + + @Override + public void countNodes(TraversalRequest request, StreamObserver responseObserver) { + AtomicInteger count = new AtomicInteger(0); + SwhBidirectionalGraph g = graph.copy(); + TraversalRequest fixedReq = TraversalRequest.newBuilder(request) + // Ignore return fields, just count nodes + .setReturnFields(NodeFields.getDefaultInstance()).build(); + Traversal.simpleTraversal(g, fixedReq, (Node node) -> { + count.incrementAndGet(); + }); + CountResponse response = CountResponse.newBuilder().setCount(count.get()).build(); + responseObserver.onNext(response); + responseObserver.onCompleted(); + } + + @Override + public void countEdges(TraversalRequest request, StreamObserver responseObserver) { + AtomicInteger count = new AtomicInteger(0); + SwhBidirectionalGraph g = graph.copy(); + TraversalRequest fixedReq = TraversalRequest.newBuilder(request) + // Force return empty successors to count the edges + .setReturnFields(NodeFields.newBuilder().setSuccessor(true).setSuccessorSwhid(false).build()) + .build(); + Traversal.simpleTraversal(g, fixedReq, (Node node) -> { + count.addAndGet(node.getSuccessorCount()); + }); + CountResponse response = CountResponse.newBuilder().setCount(count.get()).build(); + responseObserver.onNext(response); + responseObserver.onCompleted(); + } + } +} diff --git a/java/src/main/java/org/softwareheritage/graph/rpc/Traversal.java b/java/src/main/java/org/softwareheritage/graph/rpc/Traversal.java new file mode 100644 --- /dev/null +++ b/java/src/main/java/org/softwareheritage/graph/rpc/Traversal.java @@ -0,0 +1,275 @@ +package org.softwareheritage.graph.rpc; + +import com.google.protobuf.ByteString; +import it.unimi.dsi.big.webgraph.LazyLongIterator; +import it.unimi.dsi.big.webgraph.labelling.ArcLabelledNodeIterator; +import it.unimi.dsi.big.webgraph.labelling.Label; +import org.softwareheritage.graph.*; +import org.softwareheritage.graph.labels.DirEntry; + +import java.util.*; + +public class Traversal { + private static LazyLongIterator filterSuccessors(SwhUnidirectionalGraph g, long nodeId, AllowedEdges allowedEdges) { + if (allowedEdges.restrictedTo == null) { + // All edges are allowed, bypass edge check + return g.successors(nodeId); + } else { + LazyLongIterator allSuccessors = g.successors(nodeId); + return new LazyLongIterator() { + @Override + public long nextLong() { + long neighbor; + while ((neighbor = allSuccessors.nextLong()) != -1) { + if (allowedEdges.isAllowed(g.getNodeType(nodeId), g.getNodeType(neighbor))) { + return neighbor; + } + } + return -1; + } + + @Override + public long skip(final long n) { + long i = 0; + while (i < n && nextLong() != -1) + i++; + return i; + } + }; + } + } + + private static ArcLabelledNodeIterator.LabelledArcIterator filterLabelledSuccessors(SwhUnidirectionalGraph g, + long nodeId, AllowedEdges allowedEdges) { + if (allowedEdges.restrictedTo == null) { + // All edges are allowed, bypass edge check + return g.labelledSuccessors(nodeId); + } else { + ArcLabelledNodeIterator.LabelledArcIterator allSuccessors = g.labelledSuccessors(nodeId); + return new ArcLabelledNodeIterator.LabelledArcIterator() { + @Override + public Label label() { + return allSuccessors.label(); + } + + @Override + public long nextLong() { + long neighbor; + while ((neighbor = allSuccessors.nextLong()) != -1) { + if (allowedEdges.isAllowed(g.getNodeType(nodeId), g.getNodeType(neighbor))) { + return neighbor; + } + } + return -1; + } + + @Override + public long skip(final long n) { + long i = 0; + while (i < n && nextLong() != -1) + i++; + return i; + } + }; + } + } + + private static class NodeFilterChecker { + private final SwhUnidirectionalGraph g; + private final NodeFilter filter; + private final AllowedNodes allowedNodes; + + private NodeFilterChecker(SwhUnidirectionalGraph graph, NodeFilter filter) { + this.g = graph; + this.filter = filter; + this.allowedNodes = new AllowedNodes(filter.hasTypes() ? filter.getTypes() : "*"); + } + + public boolean allowed(long nodeId) { + if (filter == null) { + return true; + } + if (!this.allowedNodes.isAllowed(g.getNodeType(nodeId))) { + return false; + } + + return true; + } + } + + public static SwhUnidirectionalGraph getDirectedGraph(SwhBidirectionalGraph g, TraversalRequest request) { + switch (request.getDirection()) { + case FORWARD: + return g.getForwardGraph(); + case BACKWARD: + return g.getBackwardGraph(); + case BOTH: + return new SwhUnidirectionalGraph(g.symmetrize(), g.getProperties()); + } + throw new IllegalArgumentException("Unknown direction: " + request.getDirection()); + } + + public static void simpleTraversal(SwhBidirectionalGraph bidirectionalGraph, TraversalRequest request, + NodeObserver nodeObserver) { + SwhUnidirectionalGraph g = getDirectedGraph(bidirectionalGraph, request); + NodeFilterChecker nodeReturnChecker = new NodeFilterChecker(g, request.getReturnNodes()); + + AllowedEdges allowedEdges = new AllowedEdges(request.hasEdges() ? request.getEdges() : "*"); + + Queue queue = new ArrayDeque<>(); + HashSet visited = new HashSet<>(); + request.getSrcList().forEach(srcSwhid -> { + long srcNodeId = g.getNodeId(new SWHID(srcSwhid)); + queue.add(srcNodeId); + visited.add(srcNodeId); + }); + queue.add(-1L); // Depth sentinel + + long edgesAccessed = 0; + long currentDepth = 0; + while (!queue.isEmpty()) { + long curr = queue.poll(); + if (curr == -1L) { + ++currentDepth; + if (!queue.isEmpty()) { + queue.add(-1L); + } + continue; + } + if (request.hasMaxDepth() && currentDepth > request.getMaxDepth()) { + break; + } + edgesAccessed += g.outdegree(curr); + if (request.hasMaxEdges() && edgesAccessed >= request.getMaxEdges()) { + break; + } + + Node.Builder nodeBuilder = null; + if (nodeReturnChecker.allowed(curr) && (!request.hasMinDepth() || currentDepth >= request.getMinDepth())) { + nodeBuilder = Node.newBuilder(); + buildNodeProperties(g, request.getReturnFields(), nodeBuilder, curr); + } + + ArcLabelledNodeIterator.LabelledArcIterator it = filterLabelledSuccessors(g, curr, allowedEdges); + long traversalSuccessors = 0; + for (long succ; (succ = it.nextLong()) != -1;) { + traversalSuccessors++; + if (!visited.contains(succ)) { + queue.add(succ); + visited.add(succ); + } + buildSuccessorProperties(g, request.getReturnFields(), nodeBuilder, curr, succ, it.label()); + } + if (request.getReturnNodes().hasMinTraversalSuccessors() + && traversalSuccessors < request.getReturnNodes().getMinTraversalSuccessors() + || request.getReturnNodes().hasMaxTraversalSuccessors() + && traversalSuccessors > request.getReturnNodes().getMaxTraversalSuccessors()) { + nodeBuilder = null; + } + if (nodeBuilder != null) { + nodeObserver.onNext(nodeBuilder.build()); + } + } + } + + private static void buildNodeProperties(SwhUnidirectionalGraph graph, NodeFields fields, Node.Builder nodeBuilder, + long node) { + if (fields == null || !fields.hasSwhid() || fields.getSwhid()) { + nodeBuilder.setSwhid(graph.getSWHID(node).toString()); + } + if (fields == null) { + return; + } + + switch (graph.getNodeType(node)) { + case CNT: + if (fields.hasCntLength()) { + nodeBuilder.setCntLength(graph.getContentLength(node)); + } + if (fields.hasCntIsSkipped()) { + nodeBuilder.setCntIsSkipped(graph.isContentSkipped(node)); + } + break; + case REV: + if (fields.getRevAuthor()) { + nodeBuilder.setRevAuthor(graph.getAuthorId(node)); + } + if (fields.getRevCommitter()) { + nodeBuilder.setRevAuthor(graph.getCommitterId(node)); + } + if (fields.getRevAuthorDate()) { + nodeBuilder.setRevAuthorDate(graph.getAuthorTimestamp(node)); + } + if (fields.getRevAuthorDateOffset()) { + nodeBuilder.setRevAuthorDateOffset(graph.getAuthorTimestampOffset(node)); + } + if (fields.getRevCommitterDate()) { + nodeBuilder.setRevCommitterDate(graph.getCommitterTimestamp(node)); + } + if (fields.getRevCommitterDateOffset()) { + nodeBuilder.setRevCommitterDateOffset(graph.getCommitterTimestampOffset(node)); + } + if (fields.getRevMessage()) { + byte[] msg = graph.getMessage(node); + if (msg != null) { + nodeBuilder.setRevMessage(ByteString.copyFrom(msg)); + } + } + break; + case REL: + if (fields.getRelAuthor()) { + nodeBuilder.setRelAuthor(graph.getAuthorId(node)); + } + if (fields.getRelAuthorDate()) { + nodeBuilder.setRelAuthorDate(graph.getAuthorTimestamp(node)); + } + if (fields.getRelAuthorDateOffset()) { + nodeBuilder.setRelAuthorDateOffset(graph.getAuthorTimestampOffset(node)); + } + if (fields.getRelName()) { + byte[] msg = graph.getTagName(node); + if (msg != null) { + nodeBuilder.setRelName(ByteString.copyFrom(msg)); + } + } + if (fields.getRelMessage()) { + byte[] msg = graph.getMessage(node); + if (msg != null) { + nodeBuilder.setRelMessage(ByteString.copyFrom(msg)); + } + } + break; + case ORI: + if (fields.getOriUrl()) { + String url = graph.getUrl(node); + if (url != null) { + nodeBuilder.setOriUrl(url); + } + } + } + } + + private static void buildSuccessorProperties(SwhUnidirectionalGraph graph, NodeFields fields, + Node.Builder nodeBuilder, long src, long dst, Label label) { + if (nodeBuilder != null && fields != null && fields.getSuccessor()) { + Successor.Builder successorBuilder = Successor.newBuilder(); + if (!fields.hasSuccessorSwhid() || fields.getSuccessorSwhid()) { + successorBuilder.setSwhid(graph.getSWHID(dst).toString()); + } + if (fields.getSuccessorLabel()) { + DirEntry[] entries = (DirEntry[]) label.get(); + for (DirEntry entry : entries) { + EdgeLabel.Builder builder = EdgeLabel.newBuilder(); + builder.setName(ByteString.copyFrom(graph.getLabelName(entry.filenameId))); + builder.setPermission(entry.permission); + successorBuilder.addLabel(builder.build()); + } + } + nodeBuilder.addSuccessor(successorBuilder.build()); + } + } + + public interface NodeObserver { + void onNext(Node nodeId); + } +} diff --git a/java/src/main/java/org/softwareheritage/graph/server/App.java b/java/src/main/java/org/softwareheritage/graph/server/App.java deleted file mode 100644 --- a/java/src/main/java/org/softwareheritage/graph/server/App.java +++ /dev/null @@ -1,196 +0,0 @@ -package org.softwareheritage.graph.server; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.PropertyNamingStrategy; -import com.martiansoftware.jsap.*; -import io.javalin.Javalin; -import io.javalin.http.Context; -import io.javalin.plugin.json.JavalinJackson; -import org.softwareheritage.graph.SwhBidirectionalGraph; -import org.softwareheritage.graph.Stats; -import org.softwareheritage.graph.SWHID; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -/** - * Web framework of the swh-graph server RPC API. - * - * @author The Software Heritage developers - */ - -public class App { - /** - * Main entrypoint. - * - * @param args command line arguments - */ - public static void main(String[] args) throws IOException, JSAPException { - SimpleJSAP jsap = new SimpleJSAP(App.class.getName(), - "Server to load and query a compressed graph representation of Software Heritage archive.", - new Parameter[]{ - new FlaggedOption("port", JSAP.INTEGER_PARSER, "5009", JSAP.NOT_REQUIRED, 'p', "port", - "Binding port of the server."), - new UnflaggedOption("graphPath", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, - JSAP.NOT_GREEDY, "The basename of the compressed graph."), - new Switch("timings", 't', "timings", "Show timings in API result metadata."),}); - - JSAPResult config = jsap.parse(args); - if (jsap.messagePrinted()) { - System.exit(1); - } - - String graphPath = config.getString("graphPath"); - int port = config.getInt("port"); - boolean showTimings = config.getBoolean("timings"); - - startServer(graphPath, port, showTimings); - } - - /** - * Loads compressed graph and starts the web server to query it. - * - * @param graphPath basename of the compressed graph - * @param port binding port of the server - * @param showTimings true if timings should be in results metadata, false otherwise - */ - private static void startServer(String graphPath, int port, boolean showTimings) throws IOException { - SwhBidirectionalGraph graph = SwhBidirectionalGraph.loadMapped(graphPath); - Stats stats = new Stats(graphPath); - - // Clean up on exit - Runtime.getRuntime().addShutdownHook(new Thread() { - public void run() { - try { - graph.close(); - } catch (IOException e) { - System.out.println("Could not clean up graph on exit: " + e); - } - } - }); - - // Configure Jackson JSON to use snake case naming style - ObjectMapper objectMapper = JavalinJackson.getObjectMapper(); - objectMapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); - JavalinJackson.configure(objectMapper); - - Javalin app = Javalin.create().start(port); - - app.before("/stats/*", ctx -> { - checkQueryStrings(ctx, ""); - }); - app.before("/leaves/*", ctx -> { - checkQueryStrings(ctx, "direction|edges"); - }); - app.before("/neighbors/*", ctx -> { - checkQueryStrings(ctx, "direction|edges"); - }); - app.before("/visit/*", ctx -> { - checkQueryStrings(ctx, "direction|edges"); - }); - app.before("/walk/*", ctx -> { - checkQueryStrings(ctx, "direction|edges|traversal"); - }); - - app.get("/stats/", ctx -> { - ctx.json(stats); - }); - - // Graph traversal endpoints - // By default the traversal is a forward DFS using all edges - - app.get("/leaves/:src", ctx -> { - SWHID src = new SWHID(ctx.pathParam("src")); - String direction = ctx.queryParam("direction", "forward"); - String edgesFmt = ctx.queryParam("edges", "*"); - - Endpoint endpoint = new Endpoint(graph, direction, edgesFmt); - Endpoint.Output output = endpoint.leaves(new Endpoint.Input(src)); - ctx.json(formatEndpointOutput(output, showTimings)); - }); - - app.get("/neighbors/:src", ctx -> { - SWHID src = new SWHID(ctx.pathParam("src")); - String direction = ctx.queryParam("direction", "forward"); - String edgesFmt = ctx.queryParam("edges", "*"); - - Endpoint endpoint = new Endpoint(graph, direction, edgesFmt); - Endpoint.Output output = endpoint.neighbors(new Endpoint.Input(src)); - ctx.json(formatEndpointOutput(output, showTimings)); - }); - - app.get("/visit/nodes/:src", ctx -> { - SWHID src = new SWHID(ctx.pathParam("src")); - String direction = ctx.queryParam("direction", "forward"); - String edgesFmt = ctx.queryParam("edges", "*"); - - Endpoint endpoint = new Endpoint(graph, direction, edgesFmt); - Endpoint.Output output = endpoint.visitNodes(new Endpoint.Input(src)); - ctx.json(formatEndpointOutput(output, showTimings)); - }); - - app.get("/visit/paths/:src", ctx -> { - SWHID src = new SWHID(ctx.pathParam("src")); - String direction = ctx.queryParam("direction", "forward"); - String edgesFmt = ctx.queryParam("edges", "*"); - - Endpoint endpoint = new Endpoint(graph, direction, edgesFmt); - Endpoint.Output output = endpoint.visitPaths(new Endpoint.Input(src)); - ctx.json(formatEndpointOutput(output, showTimings)); - }); - - app.get("/walk/:src/:dst", ctx -> { - SWHID src = new SWHID(ctx.pathParam("src")); - String dstFmt = ctx.pathParam("dst"); - String direction = ctx.queryParam("direction", "forward"); - String edgesFmt = ctx.queryParam("edges", "*"); - String algorithm = ctx.queryParam("traversal", "dfs"); - - Endpoint endpoint = new Endpoint(graph, direction, edgesFmt); - Endpoint.Output output = endpoint.walk(new Endpoint.Input(src, dstFmt, algorithm)); - ctx.json(formatEndpointOutput(output, showTimings)); - }); - - app.exception(IllegalArgumentException.class, (e, ctx) -> { - ctx.status(400); - ctx.result(e.getMessage()); - }); - } - - /** - * Checks query strings names provided to the RPC API. - * - * @param ctx Javalin HTTP request context - * @param allowedFmt a regular expression describing allowed query strings names - * @throws IllegalArgumentException unknown query string provided - */ - private static void checkQueryStrings(Context ctx, String allowedFmt) { - Map> queryParamMap = ctx.queryParamMap(); - for (String key : queryParamMap.keySet()) { - if (!key.matches(allowedFmt)) { - throw new IllegalArgumentException("Unknown query string: " + key); - } - } - } - - /** - * Formats endpoint result into final JSON for the RPC API. - *

- * Removes unwanted information if necessary, such as timings (to prevent use of side channels - * attacks). - * - * @param output endpoint operation output which needs formatting - * @param showTimings true if timings should be in results metadata, false otherwise - * @return final Object with desired JSON format - */ - private static Object formatEndpointOutput(Endpoint.Output output, boolean showTimings) { - if (showTimings) { - return output; - } else { - Map metaNoTimings = Map.of("nb_edges_accessed", output.meta.nbEdgesAccessed); - Map outputNoTimings = Map.of("result", output.result, "meta", metaNoTimings); - return outputNoTimings; - } - } -} diff --git a/java/src/main/java/org/softwareheritage/graph/server/Endpoint.java b/java/src/main/java/org/softwareheritage/graph/server/Endpoint.java deleted file mode 100644 --- a/java/src/main/java/org/softwareheritage/graph/server/Endpoint.java +++ /dev/null @@ -1,309 +0,0 @@ -package org.softwareheritage.graph.server; - -import org.softwareheritage.graph.*; -import org.softwareheritage.graph.benchmark.utils.Timing; - -import java.util.ArrayList; - -/** - * RPC API endpoints wrapper functions. - *

- * Graph operations are segmented between high-level class (this one) and the low-level class - * ({@link Traversal}). The {@link Endpoint} class creates wrappers for each endpoints by performing - * all the input/output node ids conversions and logging timings. - * - * @author The Software Heritage developers - * @see Traversal - */ - -public class Endpoint { - /** Graph where traversal endpoint is performed */ - SwhBidirectionalGraph graph; - /** Internal traversal API */ - Traversal traversal; - - /** - * Constructor. - * - * @param graph the graph used for traversal endpoint - * @param direction a string (either "forward" or "backward") specifying edge orientation - * @param edgesFmt a formatted string describing allowed - * edges - */ - public Endpoint(SwhBidirectionalGraph graph, String direction, String edgesFmt) { - this.graph = graph; - this.traversal = new Traversal(graph, direction, edgesFmt); - } - - /** - * Converts a list of (internal) long node ids to a list of corresponding (external) SWHIDs. - * - * @param nodeIds the list of long node ids - * @return a list of corresponding SWHIDs - */ - private ArrayList convertNodesToSWHIDs(ArrayList nodeIds) { - ArrayList swhids = new ArrayList<>(); - for (long nodeId : nodeIds) { - swhids.add(graph.getSWHID(nodeId)); - } - return swhids; - } - - /** - * Converts a list of (internal) long node ids to the corresponding {@link SwhPath}. - * - * @param nodeIds the list of long node ids - * @return the corresponding {@link SwhPath} - * @see org.softwareheritage.graph.SwhPath - */ - private SwhPath convertNodesToSwhPath(ArrayList nodeIds) { - SwhPath path = new SwhPath(); - for (long nodeId : nodeIds) { - path.add(graph.getSWHID(nodeId)); - } - return path; - } - - /** - * Converts a list of paths made of (internal) long node ids to one made of {@link SwhPath}-s. - * - * @param pathsNodeId the list of paths with long node ids - * @return a list of corresponding {@link SwhPath} - * @see org.softwareheritage.graph.SwhPath - */ - private ArrayList convertPathsToSWHIDs(ArrayList> pathsNodeId) { - ArrayList paths = new ArrayList<>(); - for (ArrayList path : pathsNodeId) { - paths.add(convertNodesToSwhPath(path)); - } - return paths; - } - - /** - * Leaves endpoint wrapper. - * - * @param input input parameters for the underlying endpoint call - * @return the resulting list of {@link SWHID} from endpoint call and operation metadata - * @see SWHID - * @see Traversal#leaves(long) - */ - public Output leaves(Input input) { - Output> output = new Output<>(); - long startTime; - - startTime = Timing.start(); - long srcNodeId = graph.getNodeId(input.src); - output.meta.timings.swhid2node = Timing.stop(startTime); - - startTime = Timing.start(); - ArrayList nodeIds = traversal.leaves(srcNodeId); - output.meta.timings.traversal = Timing.stop(startTime); - output.meta.nbEdgesAccessed = traversal.getNbEdgesAccessed(); - - startTime = Timing.start(); - output.result = convertNodesToSWHIDs(nodeIds); - output.meta.timings.node2swhid = Timing.stop(startTime); - - return output; - } - - /** - * Neighbors endpoint wrapper. - * - * @param input input parameters for the underlying endpoint call - * @return the resulting list of {@link SWHID} from endpoint call and operation metadata - * @see SWHID - * @see Traversal#neighbors(long) - */ - public Output neighbors(Input input) { - Output> output = new Output<>(); - long startTime; - - startTime = Timing.start(); - long srcNodeId = graph.getNodeId(input.src); - output.meta.timings.swhid2node = Timing.stop(startTime); - - startTime = Timing.start(); - ArrayList nodeIds = traversal.neighbors(srcNodeId); - output.meta.timings.traversal = Timing.stop(startTime); - output.meta.nbEdgesAccessed = traversal.getNbEdgesAccessed(); - - startTime = Timing.start(); - output.result = convertNodesToSWHIDs(nodeIds); - output.meta.timings.node2swhid = Timing.stop(startTime); - - return output; - } - - /** - * Walk endpoint wrapper. - * - * @param input input parameters for the underlying endpoint call - * @return the resulting {@link SwhPath} from endpoint call and operation metadata - * @see SWHID - * @see org.softwareheritage.graph.SwhPath - * @see Traversal#walk - */ - public Output walk(Input input) { - Output output = new Output<>(); - long startTime; - - startTime = Timing.start(); - long srcNodeId = graph.getNodeId(input.src); - output.meta.timings.swhid2node = Timing.stop(startTime); - - ArrayList nodeIds = new ArrayList(); - - // Destination is either a SWHID or a node type - try { - SWHID dstSWHID = new SWHID(input.dstFmt); - long dstNodeId = graph.getNodeId(dstSWHID); - - startTime = Timing.start(); - nodeIds = traversal.walk(srcNodeId, dstNodeId, input.algorithm); - output.meta.timings.traversal = Timing.stop(startTime); - } catch (IllegalArgumentException ignored1) { - try { - Node.Type dstType = Node.Type.fromStr(input.dstFmt); - - startTime = Timing.start(); - nodeIds = traversal.walk(srcNodeId, dstType, input.algorithm); - output.meta.timings.traversal = Timing.stop(startTime); - } catch (IllegalArgumentException ignored2) { - } - } - - output.meta.nbEdgesAccessed = traversal.getNbEdgesAccessed(); - - startTime = Timing.start(); - output.result = convertNodesToSwhPath(nodeIds); - output.meta.timings.node2swhid = Timing.stop(startTime); - - return output; - } - - /** - * VisitNodes endpoint wrapper. - * - * @param input input parameters for the underlying endpoint call - * @return the resulting list of {@link SWHID} from endpoint call and operation metadata - * @see SWHID - * @see Traversal#visitNodes(long) - */ - public Output visitNodes(Input input) { - Output> output = new Output<>(); - long startTime; - - startTime = Timing.start(); - long srcNodeId = graph.getNodeId(input.src); - output.meta.timings.swhid2node = Timing.stop(startTime); - - startTime = Timing.start(); - ArrayList nodeIds = traversal.visitNodes(srcNodeId); - output.meta.timings.traversal = Timing.stop(startTime); - output.meta.nbEdgesAccessed = traversal.getNbEdgesAccessed(); - - startTime = Timing.start(); - output.result = convertNodesToSWHIDs(nodeIds); - output.meta.timings.node2swhid = Timing.stop(startTime); - - return output; - } - - /** - * VisitPaths endpoint wrapper. - * - * @param input input parameters for the underlying endpoint call - * @return the resulting list of {@link SwhPath} from endpoint call and operation metadata - * @see SWHID - * @see org.softwareheritage.graph.SwhPath - * @see Traversal#visitPaths(long) - */ - public Output visitPaths(Input input) { - Output> output = new Output<>(); - long startTime; - - startTime = Timing.start(); - long srcNodeId = graph.getNodeId(input.src); - output.meta.timings.swhid2node = Timing.stop(startTime); - - startTime = Timing.start(); - ArrayList> paths = traversal.visitPaths(srcNodeId); - output.meta.timings.traversal = Timing.stop(startTime); - output.meta.nbEdgesAccessed = traversal.getNbEdgesAccessed(); - - startTime = Timing.start(); - output.result = convertPathsToSWHIDs(paths); - output.meta.timings.node2swhid = Timing.stop(startTime); - - return output; - } - - /** - * Wrapper class to unify traversal methods input signatures. - */ - public static class Input { - /** Source node of endpoint call specified as a {@link SWHID} */ - public SWHID src; - /** - * Destination formatted string as described in the - * API - */ - public String dstFmt; - /** Traversal algorithm used in endpoint call (either "dfs" or "bfs") */ - public String algorithm; - - public Input(SWHID src) { - this.src = src; - } - - public Input(SWHID src, String dstFmt, String algorithm) { - this.src = src; - this.dstFmt = dstFmt; - this.algorithm = algorithm; - } - } - - /** - * Wrapper class to return both the endpoint result and metadata (such as timings). - */ - public static class Output { - /** The result content itself */ - public T result; - /** Various metadata about the result */ - public Meta meta; - - public Output() { - this.result = null; - this.meta = new Meta(); - } - - /** - * Endpoint result metadata. - */ - public class Meta { - /** Operations timings */ - public Timings timings; - /** Number of edges accessed during traversal */ - public long nbEdgesAccessed; - - public Meta() { - this.timings = new Timings(); - this.nbEdgesAccessed = 0; - } - - /** - * Wrapper class for JSON output format. - */ - public class Timings { - /** Time in seconds to do the traversal */ - public double traversal; - /** Time in seconds to convert input SWHID to node id */ - public double swhid2node; - /** Time in seconds to convert output node ids to SWHIDs */ - public double node2swhid; - } - } - } -} diff --git a/java/src/main/proto b/java/src/main/proto new file mode 120000 --- /dev/null +++ b/java/src/main/proto @@ -0,0 +1 @@ +../../../proto \ No newline at end of file diff --git a/java/src/test/java/org/softwareheritage/graph/GraphTest.java b/java/src/test/java/org/softwareheritage/graph/GraphTest.java --- a/java/src/test/java/org/softwareheritage/graph/GraphTest.java +++ b/java/src/test/java/org/softwareheritage/graph/GraphTest.java @@ -6,15 +6,15 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; +import java.util.Comparator; import java.util.Iterator; import com.github.luben.zstd.ZstdInputStream; import it.unimi.dsi.big.webgraph.LazyLongIterator; import it.unimi.dsi.big.webgraph.LazyLongIterators; -import org.hamcrest.MatcherAssert; import org.junit.jupiter.api.BeforeAll; -import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; +import static org.junit.Assert.assertEquals; public class GraphTest { static SwhBidirectionalGraph graph; @@ -23,11 +23,14 @@ @BeforeAll public static void setUp() throws IOException { - Path graphPath = Paths.get("..", "swh", "graph", "tests", "dataset", "compressed", "example"); - graph = SwhBidirectionalGraph.loadMapped(graphPath.toString()); + graph = SwhBidirectionalGraph.loadLabelled(getGraphPath().toString()); } - public SwhBidirectionalGraph getGraph() { + public static Path getGraphPath() { + return Paths.get("..", "swh", "graph", "tests", "dataset", "compressed", "example"); + } + + public static SwhBidirectionalGraph getGraph() { return graph; } @@ -35,8 +38,12 @@ return new SWHID(String.format("swh:1:%s:%040d", type, num)); } - public static void assertEqualsAnyOrder(Collection expecteds, Collection actuals) { - MatcherAssert.assertThat(expecteds, containsInAnyOrder(actuals.toArray())); + public static void assertEqualsAnyOrder(Collection expected, Collection actual) { + ArrayList expectedList = new ArrayList<>(expected); + ArrayList actualList = new ArrayList<>(actual); + expectedList.sort(Comparator.comparing(Object::toString)); + actualList.sort(Comparator.comparing(Object::toString)); + assertEquals(expectedList, actualList); } public static ArrayList lazyLongIteratorToList(LazyLongIterator input) { diff --git a/java/src/test/java/org/softwareheritage/graph/NeighborsTest.java b/java/src/test/java/org/softwareheritage/graph/NeighborsTest.java deleted file mode 100644 --- a/java/src/test/java/org/softwareheritage/graph/NeighborsTest.java +++ /dev/null @@ -1,141 +0,0 @@ -package org.softwareheritage.graph; - -import java.util.ArrayList; - -import org.junit.jupiter.api.Test; -import org.softwareheritage.graph.server.Endpoint; - -// Avoid warnings concerning Endpoint.Output.result manual cast -@SuppressWarnings("unchecked") -public class NeighborsTest extends GraphTest { - @Test - public void zeroNeighbor() { - SwhBidirectionalGraph graph = getGraph(); - ArrayList expectedNodes = new ArrayList<>(); - - SWHID src1 = new SWHID(TEST_ORIGIN_ID); - Endpoint endpoint1 = new Endpoint(graph, "backward", "*"); - ArrayList actuals1 = (ArrayList) endpoint1.neighbors(new Endpoint.Input(src1)).result; - GraphTest.assertEqualsAnyOrder(expectedNodes, actuals1); - - SWHID src2 = new SWHID("swh:1:cnt:0000000000000000000000000000000000000004"); - Endpoint endpoint2 = new Endpoint(graph, "forward", "*"); - ArrayList actuals2 = (ArrayList) endpoint2.neighbors(new Endpoint.Input(src2)).result; - GraphTest.assertEqualsAnyOrder(expectedNodes, actuals2); - - SWHID src3 = new SWHID("swh:1:cnt:0000000000000000000000000000000000000015"); - Endpoint endpoint3 = new Endpoint(graph, "forward", "*"); - ArrayList actuals3 = (ArrayList) endpoint3.neighbors(new Endpoint.Input(src3)).result; - GraphTest.assertEqualsAnyOrder(expectedNodes, actuals3); - - SWHID src4 = new SWHID("swh:1:rel:0000000000000000000000000000000000000019"); - Endpoint endpoint4 = new Endpoint(graph, "backward", "*"); - ArrayList actuals4 = (ArrayList) endpoint4.neighbors(new Endpoint.Input(src4)).result; - GraphTest.assertEqualsAnyOrder(expectedNodes, actuals4); - - SWHID src5 = new SWHID("swh:1:dir:0000000000000000000000000000000000000008"); - Endpoint endpoint5 = new Endpoint(graph, "forward", "snp:*,rev:*,rel:*"); - ArrayList actuals5 = (ArrayList) endpoint5.neighbors(new Endpoint.Input(src5)).result; - GraphTest.assertEqualsAnyOrder(expectedNodes, actuals5); - } - - @Test - public void oneNeighbor() { - SwhBidirectionalGraph graph = getGraph(); - - SWHID src1 = new SWHID("swh:1:rev:0000000000000000000000000000000000000003"); - Endpoint endpoint1 = new Endpoint(graph, "forward", "*"); - ArrayList expectedNodes1 = new ArrayList<>(); - expectedNodes1.add(new SWHID("swh:1:dir:0000000000000000000000000000000000000002")); - ArrayList actuals1 = (ArrayList) endpoint1.neighbors(new Endpoint.Input(src1)).result; - GraphTest.assertEqualsAnyOrder(expectedNodes1, actuals1); - - SWHID src2 = new SWHID("swh:1:dir:0000000000000000000000000000000000000017"); - Endpoint endpoint2 = new Endpoint(graph, "forward", "dir:cnt"); - ArrayList expectedNodes2 = new ArrayList<>(); - expectedNodes2.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000014")); - ArrayList actuals2 = (ArrayList) endpoint2.neighbors(new Endpoint.Input(src2)).result; - GraphTest.assertEqualsAnyOrder(expectedNodes2, actuals2); - - SWHID src3 = new SWHID("swh:1:dir:0000000000000000000000000000000000000012"); - Endpoint endpoint3 = new Endpoint(graph, "backward", "*"); - ArrayList expectedNodes3 = new ArrayList<>(); - expectedNodes3.add(new SWHID("swh:1:rev:0000000000000000000000000000000000000013")); - ArrayList actuals3 = (ArrayList) endpoint3.neighbors(new Endpoint.Input(src3)).result; - GraphTest.assertEqualsAnyOrder(expectedNodes3, actuals3); - - SWHID src4 = new SWHID("swh:1:rev:0000000000000000000000000000000000000009"); - Endpoint endpoint4 = new Endpoint(graph, "backward", "rev:rev"); - ArrayList expectedNodes4 = new ArrayList<>(); - expectedNodes4.add(new SWHID("swh:1:rev:0000000000000000000000000000000000000013")); - ArrayList actuals4 = (ArrayList) endpoint4.neighbors(new Endpoint.Input(src4)).result; - GraphTest.assertEqualsAnyOrder(expectedNodes4, actuals4); - - SWHID src5 = new SWHID("swh:1:snp:0000000000000000000000000000000000000020"); - Endpoint endpoint5 = new Endpoint(graph, "backward", "*"); - ArrayList expectedNodes5 = new ArrayList<>(); - expectedNodes5.add(new SWHID(TEST_ORIGIN_ID)); - ArrayList actuals5 = (ArrayList) endpoint5.neighbors(new Endpoint.Input(src5)).result; - GraphTest.assertEqualsAnyOrder(expectedNodes5, actuals5); - } - - @Test - public void twoNeighbors() { - SwhBidirectionalGraph graph = getGraph(); - - SWHID src1 = new SWHID("swh:1:snp:0000000000000000000000000000000000000020"); - Endpoint endpoint1 = new Endpoint(graph, "forward", "*"); - ArrayList expectedNodes1 = new ArrayList<>(); - expectedNodes1.add(new SWHID("swh:1:rel:0000000000000000000000000000000000000010")); - expectedNodes1.add(new SWHID("swh:1:rev:0000000000000000000000000000000000000009")); - ArrayList actuals1 = (ArrayList) endpoint1.neighbors(new Endpoint.Input(src1)).result; - GraphTest.assertEqualsAnyOrder(expectedNodes1, actuals1); - - SWHID src2 = new SWHID("swh:1:dir:0000000000000000000000000000000000000008"); - Endpoint endpoint2 = new Endpoint(graph, "forward", "dir:cnt"); - ArrayList expectedNodes2 = new ArrayList<>(); - expectedNodes2.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000001")); - expectedNodes2.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000007")); - ArrayList actuals2 = (ArrayList) endpoint2.neighbors(new Endpoint.Input(src2)).result; - GraphTest.assertEqualsAnyOrder(expectedNodes2, actuals2); - - SWHID src3 = new SWHID("swh:1:cnt:0000000000000000000000000000000000000001"); - Endpoint endpoint3 = new Endpoint(graph, "backward", "*"); - ArrayList expectedNodes3 = new ArrayList<>(); - expectedNodes3.add(new SWHID("swh:1:dir:0000000000000000000000000000000000000008")); - expectedNodes3.add(new SWHID("swh:1:dir:0000000000000000000000000000000000000002")); - ArrayList actuals3 = (ArrayList) endpoint3.neighbors(new Endpoint.Input(src3)).result; - GraphTest.assertEqualsAnyOrder(expectedNodes3, actuals3); - - SWHID src4 = new SWHID("swh:1:rev:0000000000000000000000000000000000000009"); - Endpoint endpoint4 = new Endpoint(graph, "backward", "rev:snp,rev:rel"); - ArrayList expectedNodes4 = new ArrayList<>(); - expectedNodes4.add(new SWHID("swh:1:snp:0000000000000000000000000000000000000020")); - expectedNodes4.add(new SWHID("swh:1:rel:0000000000000000000000000000000000000010")); - ArrayList actuals4 = (ArrayList) endpoint4.neighbors(new Endpoint.Input(src4)).result; - GraphTest.assertEqualsAnyOrder(expectedNodes4, actuals4); - } - - @Test - public void threeNeighbors() { - SwhBidirectionalGraph graph = getGraph(); - - SWHID src1 = new SWHID("swh:1:dir:0000000000000000000000000000000000000008"); - Endpoint endpoint1 = new Endpoint(graph, "forward", "*"); - ArrayList expectedNodes1 = new ArrayList<>(); - expectedNodes1.add(new SWHID("swh:1:dir:0000000000000000000000000000000000000006")); - expectedNodes1.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000001")); - expectedNodes1.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000007")); - ArrayList actuals1 = (ArrayList) endpoint1.neighbors(new Endpoint.Input(src1)).result; - GraphTest.assertEqualsAnyOrder(expectedNodes1, actuals1); - - SWHID src2 = new SWHID("swh:1:rev:0000000000000000000000000000000000000009"); - Endpoint endpoint2 = new Endpoint(graph, "backward", "*"); - ArrayList expectedNodes2 = new ArrayList<>(); - expectedNodes2.add(new SWHID("swh:1:snp:0000000000000000000000000000000000000020")); - expectedNodes2.add(new SWHID("swh:1:rel:0000000000000000000000000000000000000010")); - expectedNodes2.add(new SWHID("swh:1:rev:0000000000000000000000000000000000000013")); - ArrayList actuals2 = (ArrayList) endpoint2.neighbors(new Endpoint.Input(src2)).result; - GraphTest.assertEqualsAnyOrder(expectedNodes2, actuals2); - } -} diff --git a/java/src/test/java/org/softwareheritage/graph/VisitTest.java b/java/src/test/java/org/softwareheritage/graph/VisitTest.java deleted file mode 100644 --- a/java/src/test/java/org/softwareheritage/graph/VisitTest.java +++ /dev/null @@ -1,408 +0,0 @@ -package org.softwareheritage.graph; - -import java.util.ArrayList; -import java.util.Set; -import java.util.HashSet; - -import org.junit.jupiter.api.Test; -import org.softwareheritage.graph.server.Endpoint; - -// Avoid warnings concerning Endpoint.Output.result manual cast -@SuppressWarnings("unchecked") -public class VisitTest extends GraphTest { - private void assertSameNodesFromPaths(ArrayList paths, ArrayList nodes) { - Set expectedNodes = new HashSet(); - for (SwhPath path : paths) { - expectedNodes.addAll(path.getPath()); - } - GraphTest.assertEqualsAnyOrder(expectedNodes, nodes); - } - - @Test - public void forwardFromRoot() { - SwhBidirectionalGraph graph = getGraph(); - SWHID swhid = new SWHID(TEST_ORIGIN_ID); - Endpoint endpoint1 = new Endpoint(graph, "forward", "*"); - ArrayList paths = (ArrayList) endpoint1.visitPaths(new Endpoint.Input(swhid)).result; - Endpoint endpoint2 = new Endpoint(graph, "forward", "*"); - ArrayList nodes = (ArrayList) endpoint2.visitNodes(new Endpoint.Input(swhid)).result; - - ArrayList expectedPaths = new ArrayList(); - expectedPaths.add(new SwhPath(TEST_ORIGIN_ID, "swh:1:snp:0000000000000000000000000000000000000020", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:cnt:0000000000000000000000000000000000000007")); - expectedPaths.add(new SwhPath(TEST_ORIGIN_ID, "swh:1:snp:0000000000000000000000000000000000000020", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:cnt:0000000000000000000000000000000000000001")); - expectedPaths.add(new SwhPath(TEST_ORIGIN_ID, "swh:1:snp:0000000000000000000000000000000000000020", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:cnt:0000000000000000000000000000000000000004")); - expectedPaths.add(new SwhPath(TEST_ORIGIN_ID, "swh:1:snp:0000000000000000000000000000000000000020", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:cnt:0000000000000000000000000000000000000005")); - expectedPaths.add(new SwhPath(TEST_ORIGIN_ID, "swh:1:snp:0000000000000000000000000000000000000020", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:rev:0000000000000000000000000000000000000003", - "swh:1:dir:0000000000000000000000000000000000000002", - "swh:1:cnt:0000000000000000000000000000000000000001")); - expectedPaths.add(new SwhPath(TEST_ORIGIN_ID, "swh:1:snp:0000000000000000000000000000000000000020", - "swh:1:rel:0000000000000000000000000000000000000010", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:cnt:0000000000000000000000000000000000000007")); - expectedPaths.add(new SwhPath(TEST_ORIGIN_ID, "swh:1:snp:0000000000000000000000000000000000000020", - "swh:1:rel:0000000000000000000000000000000000000010", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:cnt:0000000000000000000000000000000000000001")); - expectedPaths.add(new SwhPath(TEST_ORIGIN_ID, "swh:1:snp:0000000000000000000000000000000000000020", - "swh:1:rel:0000000000000000000000000000000000000010", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:cnt:0000000000000000000000000000000000000004")); - expectedPaths.add(new SwhPath(TEST_ORIGIN_ID, "swh:1:snp:0000000000000000000000000000000000000020", - "swh:1:rel:0000000000000000000000000000000000000010", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:cnt:0000000000000000000000000000000000000005")); - expectedPaths.add(new SwhPath(TEST_ORIGIN_ID, "swh:1:snp:0000000000000000000000000000000000000020", - "swh:1:rel:0000000000000000000000000000000000000010", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:rev:0000000000000000000000000000000000000003", - "swh:1:dir:0000000000000000000000000000000000000002", - "swh:1:cnt:0000000000000000000000000000000000000001")); - - GraphTest.assertEqualsAnyOrder(expectedPaths, paths); - assertSameNodesFromPaths(expectedPaths, nodes); - } - - @Test - public void forwardFromMiddle() { - SwhBidirectionalGraph graph = getGraph(); - SWHID swhid = new SWHID("swh:1:dir:0000000000000000000000000000000000000012"); - Endpoint endpoint1 = new Endpoint(graph, "forward", "*"); - ArrayList paths = (ArrayList) endpoint1.visitPaths(new Endpoint.Input(swhid)).result; - Endpoint endpoint2 = new Endpoint(graph, "forward", "*"); - ArrayList nodes = (ArrayList) endpoint2.visitNodes(new Endpoint.Input(swhid)).result; - - ArrayList expectedPaths = new ArrayList(); - expectedPaths.add(new SwhPath("swh:1:dir:0000000000000000000000000000000000000012", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:cnt:0000000000000000000000000000000000000007")); - expectedPaths.add(new SwhPath("swh:1:dir:0000000000000000000000000000000000000012", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:cnt:0000000000000000000000000000000000000001")); - expectedPaths.add(new SwhPath("swh:1:dir:0000000000000000000000000000000000000012", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:cnt:0000000000000000000000000000000000000004")); - expectedPaths.add(new SwhPath("swh:1:dir:0000000000000000000000000000000000000012", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:cnt:0000000000000000000000000000000000000005")); - expectedPaths.add(new SwhPath("swh:1:dir:0000000000000000000000000000000000000012", - "swh:1:cnt:0000000000000000000000000000000000000011")); - - GraphTest.assertEqualsAnyOrder(expectedPaths, paths); - assertSameNodesFromPaths(expectedPaths, nodes); - } - - @Test - public void forwardFromLeaf() { - SwhBidirectionalGraph graph = getGraph(); - SWHID swhid = new SWHID("swh:1:cnt:0000000000000000000000000000000000000004"); - Endpoint endpoint1 = new Endpoint(graph, "forward", "*"); - ArrayList paths = (ArrayList) endpoint1.visitPaths(new Endpoint.Input(swhid)).result; - Endpoint endpoint2 = new Endpoint(graph, "forward", "*"); - ArrayList nodes = (ArrayList) endpoint2.visitNodes(new Endpoint.Input(swhid)).result; - - ArrayList expectedPaths = new ArrayList(); - expectedPaths.add(new SwhPath("swh:1:cnt:0000000000000000000000000000000000000004")); - - GraphTest.assertEqualsAnyOrder(expectedPaths, paths); - assertSameNodesFromPaths(expectedPaths, nodes); - } - - @Test - public void backwardFromRoot() { - SwhBidirectionalGraph graph = getGraph(); - SWHID swhid = new SWHID(TEST_ORIGIN_ID); - Endpoint endpoint1 = new Endpoint(graph, "backward", "*"); - ArrayList paths = (ArrayList) endpoint1.visitPaths(new Endpoint.Input(swhid)).result; - Endpoint endpoint2 = new Endpoint(graph, "backward", "*"); - ArrayList nodes = (ArrayList) endpoint2.visitNodes(new Endpoint.Input(swhid)).result; - - ArrayList expectedPaths = new ArrayList(); - expectedPaths.add(new SwhPath(TEST_ORIGIN_ID)); - - GraphTest.assertEqualsAnyOrder(expectedPaths, paths); - assertSameNodesFromPaths(expectedPaths, nodes); - } - - @Test - public void backwardFromMiddle() { - SwhBidirectionalGraph graph = getGraph(); - SWHID swhid = new SWHID("swh:1:dir:0000000000000000000000000000000000000012"); - Endpoint endpoint1 = new Endpoint(graph, "backward", "*"); - ArrayList paths = (ArrayList) endpoint1.visitPaths(new Endpoint.Input(swhid)).result; - Endpoint endpoint2 = new Endpoint(graph, "backward", "*"); - ArrayList nodes = (ArrayList) endpoint2.visitNodes(new Endpoint.Input(swhid)).result; - - ArrayList expectedPaths = new ArrayList(); - expectedPaths.add(new SwhPath("swh:1:dir:0000000000000000000000000000000000000012", - "swh:1:rev:0000000000000000000000000000000000000013", - "swh:1:rev:0000000000000000000000000000000000000018", - "swh:1:rel:0000000000000000000000000000000000000019")); - - GraphTest.assertEqualsAnyOrder(expectedPaths, paths); - assertSameNodesFromPaths(expectedPaths, nodes); - } - - @Test - public void backwardFromLeaf() { - SwhBidirectionalGraph graph = getGraph(); - SWHID swhid = new SWHID("swh:1:cnt:0000000000000000000000000000000000000004"); - Endpoint endpoint1 = new Endpoint(graph, "backward", "*"); - ArrayList paths = (ArrayList) endpoint1.visitPaths(new Endpoint.Input(swhid)).result; - Endpoint endpoint2 = new Endpoint(graph, "backward", "*"); - ArrayList nodes = (ArrayList) endpoint2.visitNodes(new Endpoint.Input(swhid)).result; - - ArrayList expectedPaths = new ArrayList(); - expectedPaths.add(new SwhPath("swh:1:cnt:0000000000000000000000000000000000000004", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:dir:0000000000000000000000000000000000000012", - "swh:1:rev:0000000000000000000000000000000000000013", - "swh:1:rev:0000000000000000000000000000000000000018", - "swh:1:rel:0000000000000000000000000000000000000019")); - expectedPaths.add(new SwhPath("swh:1:cnt:0000000000000000000000000000000000000004", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:rev:0000000000000000000000000000000000000013", - "swh:1:rev:0000000000000000000000000000000000000018", - "swh:1:rel:0000000000000000000000000000000000000019")); - expectedPaths.add(new SwhPath("swh:1:cnt:0000000000000000000000000000000000000004", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:snp:0000000000000000000000000000000000000020", TEST_ORIGIN_ID)); - expectedPaths.add(new SwhPath("swh:1:cnt:0000000000000000000000000000000000000004", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:rel:0000000000000000000000000000000000000010", - "swh:1:snp:0000000000000000000000000000000000000020", TEST_ORIGIN_ID)); - - GraphTest.assertEqualsAnyOrder(expectedPaths, paths); - assertSameNodesFromPaths(expectedPaths, nodes); - } - - @Test - public void forwardSnpToRev() { - SwhBidirectionalGraph graph = getGraph(); - SWHID swhid = new SWHID("swh:1:snp:0000000000000000000000000000000000000020"); - Endpoint endpoint1 = new Endpoint(graph, "forward", "snp:rev"); - ArrayList paths = (ArrayList) endpoint1.visitPaths(new Endpoint.Input(swhid)).result; - Endpoint endpoint2 = new Endpoint(graph, "forward", "snp:rev"); - ArrayList nodes = (ArrayList) endpoint2.visitNodes(new Endpoint.Input(swhid)).result; - - ArrayList expectedPaths = new ArrayList(); - expectedPaths.add(new SwhPath("swh:1:snp:0000000000000000000000000000000000000020", - "swh:1:rev:0000000000000000000000000000000000000009")); - - GraphTest.assertEqualsAnyOrder(expectedPaths, paths); - assertSameNodesFromPaths(expectedPaths, nodes); - } - - @Test - public void forwardRelToRevRevToRev() { - SwhBidirectionalGraph graph = getGraph(); - SWHID swhid = new SWHID("swh:1:rel:0000000000000000000000000000000000000010"); - Endpoint endpoint1 = new Endpoint(graph, "forward", "rel:rev,rev:rev"); - ArrayList paths = (ArrayList) endpoint1.visitPaths(new Endpoint.Input(swhid)).result; - Endpoint endpoint2 = new Endpoint(graph, "forward", "rel:rev,rev:rev"); - ArrayList nodes = (ArrayList) endpoint2.visitNodes(new Endpoint.Input(swhid)).result; - - ArrayList expectedPaths = new ArrayList(); - expectedPaths.add(new SwhPath("swh:1:rel:0000000000000000000000000000000000000010", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:rev:0000000000000000000000000000000000000003")); - - GraphTest.assertEqualsAnyOrder(expectedPaths, paths); - assertSameNodesFromPaths(expectedPaths, nodes); - } - - @Test - public void forwardRevToAllDirToAll() { - SwhBidirectionalGraph graph = getGraph(); - SWHID swhid = new SWHID("swh:1:rev:0000000000000000000000000000000000000013"); - Endpoint endpoint1 = new Endpoint(graph, "forward", "rev:*,dir:*"); - ArrayList paths = (ArrayList) endpoint1.visitPaths(new Endpoint.Input(swhid)).result; - Endpoint endpoint2 = new Endpoint(graph, "forward", "rev:*,dir:*"); - ArrayList nodes = (ArrayList) endpoint2.visitNodes(new Endpoint.Input(swhid)).result; - - ArrayList expectedPaths = new ArrayList(); - expectedPaths.add(new SwhPath("swh:1:rev:0000000000000000000000000000000000000013", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:cnt:0000000000000000000000000000000000000005")); - expectedPaths.add(new SwhPath("swh:1:rev:0000000000000000000000000000000000000013", - "swh:1:dir:0000000000000000000000000000000000000012", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:cnt:0000000000000000000000000000000000000005")); - expectedPaths.add(new SwhPath("swh:1:rev:0000000000000000000000000000000000000013", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:cnt:0000000000000000000000000000000000000004")); - expectedPaths.add(new SwhPath("swh:1:rev:0000000000000000000000000000000000000013", - "swh:1:dir:0000000000000000000000000000000000000012", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:cnt:0000000000000000000000000000000000000004")); - expectedPaths.add(new SwhPath("swh:1:rev:0000000000000000000000000000000000000013", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:cnt:0000000000000000000000000000000000000007")); - expectedPaths.add(new SwhPath("swh:1:rev:0000000000000000000000000000000000000013", - "swh:1:dir:0000000000000000000000000000000000000012", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:cnt:0000000000000000000000000000000000000007")); - expectedPaths.add(new SwhPath("swh:1:rev:0000000000000000000000000000000000000013", - "swh:1:dir:0000000000000000000000000000000000000012", - "swh:1:cnt:0000000000000000000000000000000000000011")); - expectedPaths.add(new SwhPath("swh:1:rev:0000000000000000000000000000000000000013", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:rev:0000000000000000000000000000000000000003", - "swh:1:dir:0000000000000000000000000000000000000002", - "swh:1:cnt:0000000000000000000000000000000000000001")); - expectedPaths.add(new SwhPath("swh:1:rev:0000000000000000000000000000000000000013", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:cnt:0000000000000000000000000000000000000001")); - expectedPaths.add(new SwhPath("swh:1:rev:0000000000000000000000000000000000000013", - "swh:1:dir:0000000000000000000000000000000000000012", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:cnt:0000000000000000000000000000000000000001")); - - GraphTest.assertEqualsAnyOrder(expectedPaths, paths); - assertSameNodesFromPaths(expectedPaths, nodes); - } - - @Test - public void forwardSnpToAllRevToAll() { - SwhBidirectionalGraph graph = getGraph(); - SWHID swhid = new SWHID("swh:1:snp:0000000000000000000000000000000000000020"); - Endpoint endpoint1 = new Endpoint(graph, "forward", "snp:*,rev:*"); - ArrayList paths = (ArrayList) endpoint1.visitPaths(new Endpoint.Input(swhid)).result; - Endpoint endpoint2 = new Endpoint(graph, "forward", "snp:*,rev:*"); - ArrayList nodes = (ArrayList) endpoint2.visitNodes(new Endpoint.Input(swhid)).result; - - ArrayList expectedPaths = new ArrayList(); - expectedPaths.add(new SwhPath("swh:1:snp:0000000000000000000000000000000000000020", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:rev:0000000000000000000000000000000000000003", - "swh:1:dir:0000000000000000000000000000000000000002")); - expectedPaths.add(new SwhPath("swh:1:snp:0000000000000000000000000000000000000020", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008")); - expectedPaths.add(new SwhPath("swh:1:snp:0000000000000000000000000000000000000020", - "swh:1:rel:0000000000000000000000000000000000000010")); - - GraphTest.assertEqualsAnyOrder(expectedPaths, paths); - assertSameNodesFromPaths(expectedPaths, nodes); - } - - @Test - public void forwardNoEdges() { - SwhBidirectionalGraph graph = getGraph(); - SWHID swhid = new SWHID("swh:1:snp:0000000000000000000000000000000000000020"); - Endpoint endpoint1 = new Endpoint(graph, "forward", ""); - ArrayList paths = (ArrayList) endpoint1.visitPaths(new Endpoint.Input(swhid)).result; - Endpoint endpoint2 = new Endpoint(graph, "forward", ""); - ArrayList nodes = (ArrayList) endpoint2.visitNodes(new Endpoint.Input(swhid)).result; - - ArrayList expectedPaths = new ArrayList(); - expectedPaths.add(new SwhPath("swh:1:snp:0000000000000000000000000000000000000020")); - - GraphTest.assertEqualsAnyOrder(expectedPaths, paths); - assertSameNodesFromPaths(expectedPaths, nodes); - } - - @Test - public void backwardRevToRevRevToRel() { - SwhBidirectionalGraph graph = getGraph(); - SWHID swhid = new SWHID("swh:1:rev:0000000000000000000000000000000000000003"); - Endpoint endpoint1 = new Endpoint(graph, "backward", "rev:rev,rev:rel"); - ArrayList paths = (ArrayList) endpoint1.visitPaths(new Endpoint.Input(swhid)).result; - Endpoint endpoint2 = new Endpoint(graph, "backward", "rev:rev,rev:rel"); - ArrayList nodes = (ArrayList) endpoint2.visitNodes(new Endpoint.Input(swhid)).result; - - ArrayList expectedPaths = new ArrayList(); - expectedPaths.add(new SwhPath("swh:1:rev:0000000000000000000000000000000000000003", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:rev:0000000000000000000000000000000000000013", - "swh:1:rev:0000000000000000000000000000000000000018", - "swh:1:rel:0000000000000000000000000000000000000019")); - expectedPaths.add(new SwhPath("swh:1:rev:0000000000000000000000000000000000000003", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:rel:0000000000000000000000000000000000000010")); - - GraphTest.assertEqualsAnyOrder(expectedPaths, paths); - assertSameNodesFromPaths(expectedPaths, nodes); - } - - @Test - public void forwardFromRootNodesOnly() { - SwhBidirectionalGraph graph = getGraph(); - SWHID swhid = new SWHID(TEST_ORIGIN_ID); - Endpoint endpoint = new Endpoint(graph, "forward", "*"); - ArrayList nodes = (ArrayList) endpoint.visitNodes(new Endpoint.Input(swhid)).result; - - ArrayList expectedNodes = new ArrayList(); - expectedNodes.add(new SWHID(TEST_ORIGIN_ID)); - expectedNodes.add(new SWHID("swh:1:snp:0000000000000000000000000000000000000020")); - expectedNodes.add(new SWHID("swh:1:rel:0000000000000000000000000000000000000010")); - expectedNodes.add(new SWHID("swh:1:rev:0000000000000000000000000000000000000009")); - expectedNodes.add(new SWHID("swh:1:rev:0000000000000000000000000000000000000003")); - expectedNodes.add(new SWHID("swh:1:dir:0000000000000000000000000000000000000002")); - expectedNodes.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000001")); - expectedNodes.add(new SWHID("swh:1:dir:0000000000000000000000000000000000000008")); - expectedNodes.add(new SWHID("swh:1:dir:0000000000000000000000000000000000000006")); - expectedNodes.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000004")); - expectedNodes.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000005")); - expectedNodes.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000007")); - - GraphTest.assertEqualsAnyOrder(expectedNodes, nodes); - } - - @Test - public void backwardRevToAllNodesOnly() { - SwhBidirectionalGraph graph = getGraph(); - SWHID swhid = new SWHID("swh:1:rev:0000000000000000000000000000000000000003"); - Endpoint endpoint = new Endpoint(graph, "backward", "rev:*"); - ArrayList nodes = (ArrayList) endpoint.visitNodes(new Endpoint.Input(swhid)).result; - - ArrayList expectedNodes = new ArrayList(); - expectedNodes.add(new SWHID("swh:1:rev:0000000000000000000000000000000000000003")); - expectedNodes.add(new SWHID("swh:1:rev:0000000000000000000000000000000000000009")); - expectedNodes.add(new SWHID("swh:1:snp:0000000000000000000000000000000000000020")); - expectedNodes.add(new SWHID("swh:1:rel:0000000000000000000000000000000000000010")); - expectedNodes.add(new SWHID("swh:1:rev:0000000000000000000000000000000000000013")); - expectedNodes.add(new SWHID("swh:1:rev:0000000000000000000000000000000000000018")); - expectedNodes.add(new SWHID("swh:1:rel:0000000000000000000000000000000000000019")); - - GraphTest.assertEqualsAnyOrder(expectedNodes, nodes); - } -} diff --git a/java/src/test/java/org/softwareheritage/graph/WalkTest.java b/java/src/test/java/org/softwareheritage/graph/WalkTest.java deleted file mode 100644 --- a/java/src/test/java/org/softwareheritage/graph/WalkTest.java +++ /dev/null @@ -1,187 +0,0 @@ -package org.softwareheritage.graph; - -import java.util.Arrays; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.softwareheritage.graph.server.Endpoint; - -public class WalkTest extends GraphTest { - @Test - public void forwardRootToLeaf() { - SwhBidirectionalGraph graph = getGraph(); - SWHID src = new SWHID("swh:1:snp:0000000000000000000000000000000000000020"); - String dstFmt = "swh:1:cnt:0000000000000000000000000000000000000005"; - - SwhPath solution1 = new SwhPath("swh:1:snp:0000000000000000000000000000000000000020", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:cnt:0000000000000000000000000000000000000005"); - SwhPath solution2 = new SwhPath("swh:1:snp:0000000000000000000000000000000000000020", - "swh:1:rel:0000000000000000000000000000000000000010", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:cnt:0000000000000000000000000000000000000005"); - - Endpoint endpoint1 = new Endpoint(graph, "forward", "*"); - SwhPath dfsPath = (SwhPath) endpoint1.walk(new Endpoint.Input(src, dstFmt, "dfs")).result; - Endpoint endpoint2 = new Endpoint(graph, "forward", "*"); - SwhPath bfsPath = (SwhPath) endpoint2.walk(new Endpoint.Input(src, dstFmt, "bfs")).result; - - List possibleSolutions = Arrays.asList(solution1, solution2); - Assertions.assertTrue(possibleSolutions.contains(dfsPath)); - Assertions.assertTrue(possibleSolutions.contains(bfsPath)); - } - - @Test - public void forwardLeafToLeaf() { - SwhBidirectionalGraph graph = getGraph(); - SWHID src = new SWHID("swh:1:cnt:0000000000000000000000000000000000000007"); - String dstFmt = "cnt"; - - SwhPath expectedPath = new SwhPath("swh:1:cnt:0000000000000000000000000000000000000007"); - - Endpoint endpoint1 = new Endpoint(graph, "forward", "*"); - SwhPath dfsPath = (SwhPath) endpoint1.walk(new Endpoint.Input(src, dstFmt, "dfs")).result; - Endpoint endpoint2 = new Endpoint(graph, "forward", "*"); - SwhPath bfsPath = (SwhPath) endpoint2.walk(new Endpoint.Input(src, dstFmt, "bfs")).result; - - Assertions.assertEquals(dfsPath, expectedPath); - Assertions.assertEquals(bfsPath, expectedPath); - } - - @Test - public void forwardRevToRev() { - SwhBidirectionalGraph graph = getGraph(); - SWHID src = new SWHID("swh:1:rev:0000000000000000000000000000000000000018"); - String dstFmt = "swh:1:rev:0000000000000000000000000000000000000003"; - - SwhPath expectedPath = new SwhPath("swh:1:rev:0000000000000000000000000000000000000018", - "swh:1:rev:0000000000000000000000000000000000000013", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:rev:0000000000000000000000000000000000000003"); - - Endpoint endpoint1 = new Endpoint(graph, "forward", "rev:rev"); - SwhPath dfsPath = (SwhPath) endpoint1.walk(new Endpoint.Input(src, dstFmt, "dfs")).result; - Endpoint endpoint2 = new Endpoint(graph, "forward", "rev:rev"); - SwhPath bfsPath = (SwhPath) endpoint2.walk(new Endpoint.Input(src, dstFmt, "bfs")).result; - - Assertions.assertEquals(dfsPath, expectedPath); - Assertions.assertEquals(bfsPath, expectedPath); - } - - @Test - public void backwardRevToRev() { - SwhBidirectionalGraph graph = getGraph(); - SWHID src = new SWHID("swh:1:rev:0000000000000000000000000000000000000003"); - String dstFmt = "swh:1:rev:0000000000000000000000000000000000000018"; - - SwhPath expectedPath = new SwhPath("swh:1:rev:0000000000000000000000000000000000000003", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:rev:0000000000000000000000000000000000000013", - "swh:1:rev:0000000000000000000000000000000000000018"); - - Endpoint endpoint1 = new Endpoint(graph, "backward", "rev:rev"); - SwhPath dfsPath = (SwhPath) endpoint1.walk(new Endpoint.Input(src, dstFmt, "dfs")).result; - Endpoint endpoint2 = new Endpoint(graph, "backward", "rev:rev"); - SwhPath bfsPath = (SwhPath) endpoint2.walk(new Endpoint.Input(src, dstFmt, "bfs")).result; - - Assertions.assertEquals(dfsPath, expectedPath); - Assertions.assertEquals(bfsPath, expectedPath); - } - - @Test - public void backwardCntToFirstSnp() { - SwhBidirectionalGraph graph = getGraph(); - SWHID src = new SWHID("swh:1:cnt:0000000000000000000000000000000000000001"); - String dstFmt = "snp"; - - SwhPath solution1 = new SwhPath("swh:1:cnt:0000000000000000000000000000000000000001", - "swh:1:dir:0000000000000000000000000000000000000002", - "swh:1:rev:0000000000000000000000000000000000000003", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:snp:0000000000000000000000000000000000000020"); - SwhPath solution2 = new SwhPath("swh:1:cnt:0000000000000000000000000000000000000001", - "swh:1:dir:0000000000000000000000000000000000000002", - "swh:1:rev:0000000000000000000000000000000000000003", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:rel:0000000000000000000000000000000000000010", - "swh:1:snp:0000000000000000000000000000000000000020"); - SwhPath solution3 = new SwhPath("swh:1:cnt:0000000000000000000000000000000000000001", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:snp:0000000000000000000000000000000000000020"); - SwhPath solution4 = new SwhPath("swh:1:cnt:0000000000000000000000000000000000000001", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:rel:0000000000000000000000000000000000000010", - "swh:1:snp:0000000000000000000000000000000000000020"); - - Endpoint endpoint1 = new Endpoint(graph, "backward", "*"); - SwhPath dfsPath = (SwhPath) endpoint1.walk(new Endpoint.Input(src, dstFmt, "dfs")).result; - Endpoint endpoint2 = new Endpoint(graph, "backward", "*"); - SwhPath bfsPath = (SwhPath) endpoint2.walk(new Endpoint.Input(src, dstFmt, "bfs")).result; - - List possibleSolutions = Arrays.asList(solution1, solution2, solution3, solution4); - Assertions.assertTrue(possibleSolutions.contains(dfsPath)); - Assertions.assertTrue(possibleSolutions.contains(bfsPath)); - } - - @Test - public void forwardRevToFirstCnt() { - SwhBidirectionalGraph graph = getGraph(); - SWHID src = new SWHID("swh:1:rev:0000000000000000000000000000000000000009"); - String dstFmt = "cnt"; - - SwhPath solution1 = new SwhPath("swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:cnt:0000000000000000000000000000000000000007"); - SwhPath solution2 = new SwhPath("swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:cnt:0000000000000000000000000000000000000005"); - SwhPath solution3 = new SwhPath("swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:dir:0000000000000000000000000000000000000006", - "swh:1:cnt:0000000000000000000000000000000000000004"); - SwhPath solution4 = new SwhPath("swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:dir:0000000000000000000000000000000000000008", - "swh:1:cnt:0000000000000000000000000000000000000001"); - SwhPath solution5 = new SwhPath("swh:1:rev:0000000000000000000000000000000000000009", - "swh:1:rev:0000000000000000000000000000000000000003", - "swh:1:dir:0000000000000000000000000000000000000002", - "swh:1:cnt:0000000000000000000000000000000000000001"); - - Endpoint endpoint1 = new Endpoint(graph, "forward", "rev:*,dir:*"); - SwhPath dfsPath = (SwhPath) endpoint1.walk(new Endpoint.Input(src, dstFmt, "dfs")).result; - Endpoint endpoint2 = new Endpoint(graph, "forward", "rev:*,dir:*"); - SwhPath bfsPath = (SwhPath) endpoint2.walk(new Endpoint.Input(src, dstFmt, "bfs")).result; - - List possibleSolutions = Arrays.asList(solution1, solution2, solution3, solution4, solution5); - Assertions.assertTrue(possibleSolutions.contains(dfsPath)); - Assertions.assertTrue(possibleSolutions.contains(bfsPath)); - } - - @Test - public void backwardDirToFirstRel() { - SwhBidirectionalGraph graph = getGraph(); - SWHID src = new SWHID("swh:1:dir:0000000000000000000000000000000000000016"); - String dstFmt = "rel"; - - SwhPath expectedPath = new SwhPath("swh:1:dir:0000000000000000000000000000000000000016", - "swh:1:dir:0000000000000000000000000000000000000017", - "swh:1:rev:0000000000000000000000000000000000000018", - "swh:1:rel:0000000000000000000000000000000000000019"); - - Endpoint endpoint1 = new Endpoint(graph, "backward", "dir:dir,dir:rev,rev:*"); - SwhPath dfsPath = (SwhPath) endpoint1.walk(new Endpoint.Input(src, dstFmt, "dfs")).result; - Endpoint endpoint2 = new Endpoint(graph, "backward", "dir:dir,dir:rev,rev:*"); - SwhPath bfsPath = (SwhPath) endpoint2.walk(new Endpoint.Input(src, dstFmt, "bfs")).result; - - Assertions.assertEquals(dfsPath, expectedPath); - Assertions.assertEquals(bfsPath, expectedPath); - } -} diff --git a/java/src/test/java/org/softwareheritage/graph/rpc/TraversalServiceTest.java b/java/src/test/java/org/softwareheritage/graph/rpc/TraversalServiceTest.java new file mode 100644 --- /dev/null +++ b/java/src/test/java/org/softwareheritage/graph/rpc/TraversalServiceTest.java @@ -0,0 +1,48 @@ +package org.softwareheritage.graph.rpc; + +import io.grpc.ManagedChannel; +import io.grpc.Server; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import io.grpc.testing.GrpcCleanupRule; +import org.junit.Rule; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.softwareheritage.graph.GraphTest; +import org.softwareheritage.graph.SWHID; + +import java.util.ArrayList; +import java.util.Iterator; + +public class TraversalServiceTest extends GraphTest { + @Rule + public final GrpcCleanupRule grpcCleanup = new GrpcCleanupRule(); + + private static Server server; + private static ManagedChannel channel; + protected static TraversalServiceGrpc.TraversalServiceBlockingStub client; + + @BeforeAll + static void setup() throws Exception { + String serverName = InProcessServerBuilder.generateName(); + assert getGraph() != null; + server = InProcessServerBuilder.forName(serverName).directExecutor() + .addService(new GraphServer.TraversalService(getGraph())).build().start(); + channel = InProcessChannelBuilder.forName(serverName).directExecutor().build(); + client = TraversalServiceGrpc.newBlockingStub(channel); + } + + @AfterAll + static void teardown() throws Exception { + channel.shutdownNow(); + server.shutdownNow(); + } + + public ArrayList getSWHIDs(Iterator it) { + ArrayList actualLeaves = new ArrayList<>(); + it.forEachRemaining((Node n) -> { + actualLeaves.add(new SWHID(n.getSwhid())); + }); + return actualLeaves; + } +} diff --git a/java/src/test/java/org/softwareheritage/graph/LeavesTest.java b/java/src/test/java/org/softwareheritage/graph/rpc/TraverseLeavesTest.java rename from java/src/test/java/org/softwareheritage/graph/LeavesTest.java rename to java/src/test/java/org/softwareheritage/graph/rpc/TraverseLeavesTest.java --- a/java/src/test/java/org/softwareheritage/graph/LeavesTest.java +++ b/java/src/test/java/org/softwareheritage/graph/rpc/TraverseLeavesTest.java @@ -1,18 +1,20 @@ -package org.softwareheritage.graph; +package org.softwareheritage.graph.rpc; + +import org.junit.jupiter.api.Test; +import org.softwareheritage.graph.GraphTest; +import org.softwareheritage.graph.SWHID; import java.util.ArrayList; -import org.junit.jupiter.api.Test; -import org.softwareheritage.graph.server.Endpoint; +public class TraverseLeavesTest extends TraversalServiceTest { + private TraversalRequest.Builder getLeavesRequestBuilder(SWHID src) { + return TraversalRequest.newBuilder().addSrc(src.toString()) + .setReturnNodes(NodeFilter.newBuilder().setMaxTraversalSuccessors(0).build()); + } -// Avoid warnings concerning Endpoint.Output.result manual cast -@SuppressWarnings("unchecked") -public class LeavesTest extends GraphTest { @Test public void forwardFromSnp() { - SwhBidirectionalGraph graph = getGraph(); - SWHID src = new SWHID("swh:1:snp:0000000000000000000000000000000000000020"); - Endpoint endpoint = new Endpoint(graph, "forward", "*"); + TraversalRequest request = getLeavesRequestBuilder(fakeSWHID("snp", 20)).build(); ArrayList expectedLeaves = new ArrayList<>(); expectedLeaves.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000001")); @@ -20,16 +22,14 @@ expectedLeaves.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000005")); expectedLeaves.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000007")); - ArrayList actualLeaves = (ArrayList) endpoint.leaves(new Endpoint.Input(src)).result; + ArrayList actualLeaves = getSWHIDs(client.traverse(request)); GraphTest.assertEqualsAnyOrder(expectedLeaves, actualLeaves); } @Test public void forwardFromRel() { - SwhBidirectionalGraph graph = getGraph(); - SWHID src = new SWHID("swh:1:rel:0000000000000000000000000000000000000019"); - Endpoint endpoint = new Endpoint(graph, "forward", "*"); - + TraversalRequest request = getLeavesRequestBuilder(fakeSWHID("rel", 19)).build(); + ArrayList actualLeaves = getSWHIDs(client.traverse(request)); ArrayList expectedLeaves = new ArrayList<>(); expectedLeaves.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000015")); expectedLeaves.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000014")); @@ -39,69 +39,55 @@ expectedLeaves.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000007")); expectedLeaves.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000011")); - ArrayList actualLeaves = (ArrayList) endpoint.leaves(new Endpoint.Input(src)).result; GraphTest.assertEqualsAnyOrder(expectedLeaves, actualLeaves); } @Test public void backwardFromLeaf() { - SwhBidirectionalGraph graph = getGraph(); - - Endpoint endpoint1 = new Endpoint(graph, "backward", "*"); - SWHID src1 = new SWHID("swh:1:cnt:0000000000000000000000000000000000000015"); + TraversalRequest request1 = getLeavesRequestBuilder(fakeSWHID("cnt", 15)).setDirection(GraphDirection.BACKWARD) + .build(); + ArrayList actualLeaves1 = getSWHIDs(client.traverse(request1)); ArrayList expectedLeaves1 = new ArrayList<>(); expectedLeaves1.add(new SWHID("swh:1:rel:0000000000000000000000000000000000000019")); - ArrayList actualLeaves1 = (ArrayList) endpoint1.leaves(new Endpoint.Input(src1)).result; GraphTest.assertEqualsAnyOrder(expectedLeaves1, actualLeaves1); - Endpoint endpoint2 = new Endpoint(graph, "backward", "*"); - SWHID src2 = new SWHID("swh:1:cnt:0000000000000000000000000000000000000004"); + TraversalRequest request2 = getLeavesRequestBuilder(fakeSWHID("cnt", 4)).setDirection(GraphDirection.BACKWARD) + .build(); + ArrayList actualLeaves2 = getSWHIDs(client.traverse(request2)); ArrayList expectedLeaves2 = new ArrayList<>(); expectedLeaves2.add(new SWHID(TEST_ORIGIN_ID)); expectedLeaves2.add(new SWHID("swh:1:rel:0000000000000000000000000000000000000019")); - ArrayList actualLeaves2 = (ArrayList) endpoint2.leaves(new Endpoint.Input(src2)).result; GraphTest.assertEqualsAnyOrder(expectedLeaves2, actualLeaves2); } @Test public void forwardRevToRevOnly() { - SwhBidirectionalGraph graph = getGraph(); - SWHID src = new SWHID("swh:1:rev:0000000000000000000000000000000000000018"); - Endpoint endpoint = new Endpoint(graph, "forward", "rev:rev"); - + TraversalRequest request = getLeavesRequestBuilder(fakeSWHID("rev", 18)).setEdges("rev:rev").build(); + ArrayList actualLeaves = getSWHIDs(client.traverse(request)); ArrayList expectedLeaves = new ArrayList<>(); expectedLeaves.add(new SWHID("swh:1:rev:0000000000000000000000000000000000000003")); - - ArrayList actualLeaves = (ArrayList) endpoint.leaves(new Endpoint.Input(src)).result; GraphTest.assertEqualsAnyOrder(expectedLeaves, actualLeaves); } @Test public void forwardDirToAll() { - SwhBidirectionalGraph graph = getGraph(); - SWHID src = new SWHID("swh:1:dir:0000000000000000000000000000000000000008"); - Endpoint endpoint = new Endpoint(graph, "forward", "dir:*"); - + TraversalRequest request = getLeavesRequestBuilder(fakeSWHID("dir", 8)).setEdges("dir:*").build(); + ArrayList actualLeaves = getSWHIDs(client.traverse(request)); ArrayList expectedLeaves = new ArrayList<>(); expectedLeaves.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000004")); expectedLeaves.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000005")); expectedLeaves.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000001")); expectedLeaves.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000007")); - - ArrayList actualLeaves = (ArrayList) endpoint.leaves(new Endpoint.Input(src)).result; GraphTest.assertEqualsAnyOrder(expectedLeaves, actualLeaves); } @Test public void backwardCntToDirDirToDir() { - SwhBidirectionalGraph graph = getGraph(); - SWHID src = new SWHID("swh:1:cnt:0000000000000000000000000000000000000005"); - Endpoint endpoint = new Endpoint(graph, "backward", "cnt:dir,dir:dir"); - + TraversalRequest request = getLeavesRequestBuilder(fakeSWHID("cnt", 5)).setEdges("cnt:dir,dir:dir") + .setDirection(GraphDirection.BACKWARD).build(); + ArrayList actualLeaves = getSWHIDs(client.traverse(request)); ArrayList expectedLeaves = new ArrayList<>(); expectedLeaves.add(new SWHID("swh:1:dir:0000000000000000000000000000000000000012")); - - ArrayList actualLeaves = (ArrayList) endpoint.leaves(new Endpoint.Input(src)).result; GraphTest.assertEqualsAnyOrder(expectedLeaves, actualLeaves); } } diff --git a/java/src/test/java/org/softwareheritage/graph/rpc/TraverseNeighborsTest.java b/java/src/test/java/org/softwareheritage/graph/rpc/TraverseNeighborsTest.java new file mode 100644 --- /dev/null +++ b/java/src/test/java/org/softwareheritage/graph/rpc/TraverseNeighborsTest.java @@ -0,0 +1,130 @@ +package org.softwareheritage.graph.rpc; + +import org.junit.jupiter.api.Test; +import org.softwareheritage.graph.GraphTest; +import org.softwareheritage.graph.SWHID; + +import java.util.ArrayList; + +public class TraverseNeighborsTest extends TraversalServiceTest { + private TraversalRequest.Builder getNeighborsRequestBuilder(SWHID src) { + return TraversalRequest.newBuilder().addSrc(src.toString()).setMinDepth(1).setMaxDepth(1); + } + + @Test + public void zeroNeighbor() { + ArrayList expectedNodes = new ArrayList<>(); + + TraversalRequest request1 = getNeighborsRequestBuilder(new SWHID(TEST_ORIGIN_ID)) + .setDirection(GraphDirection.BACKWARD).build(); + ArrayList actuals1 = getSWHIDs(client.traverse(request1)); + GraphTest.assertEqualsAnyOrder(expectedNodes, actuals1); + + TraversalRequest request2 = getNeighborsRequestBuilder(fakeSWHID("cnt", 4)).build(); + ArrayList actuals2 = getSWHIDs(client.traverse(request2)); + GraphTest.assertEqualsAnyOrder(expectedNodes, actuals2); + + TraversalRequest request3 = getNeighborsRequestBuilder(fakeSWHID("cnt", 15)).build(); + ArrayList actuals3 = getSWHIDs(client.traverse(request3)); + GraphTest.assertEqualsAnyOrder(expectedNodes, actuals3); + + TraversalRequest request4 = getNeighborsRequestBuilder(fakeSWHID("rel", 19)) + .setDirection(GraphDirection.BACKWARD).build(); + ArrayList actuals4 = getSWHIDs(client.traverse(request4)); + GraphTest.assertEqualsAnyOrder(expectedNodes, actuals4); + + TraversalRequest request5 = getNeighborsRequestBuilder(fakeSWHID("dir", 8)).setEdges("snp:*,rev:*,rel:*") + .build(); + ArrayList actuals5 = getSWHIDs(client.traverse(request5)); + GraphTest.assertEqualsAnyOrder(expectedNodes, actuals5); + } + + @Test + public void oneNeighbor() { + TraversalRequest request1 = getNeighborsRequestBuilder(fakeSWHID("rev", 3)).build(); + ArrayList actuals1 = getSWHIDs(client.traverse(request1)); + ArrayList expectedNodes1 = new ArrayList<>(); + expectedNodes1.add(new SWHID("swh:1:dir:0000000000000000000000000000000000000002")); + GraphTest.assertEqualsAnyOrder(expectedNodes1, actuals1); + + TraversalRequest request2 = getNeighborsRequestBuilder(fakeSWHID("dir", 17)).setEdges("dir:cnt").build(); + ArrayList actuals2 = getSWHIDs(client.traverse(request2)); + ArrayList expectedNodes2 = new ArrayList<>(); + expectedNodes2.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000014")); + GraphTest.assertEqualsAnyOrder(expectedNodes2, actuals2); + + TraversalRequest request3 = getNeighborsRequestBuilder(fakeSWHID("dir", 12)) + .setDirection(GraphDirection.BACKWARD).build(); + ArrayList actuals3 = getSWHIDs(client.traverse(request3)); + ArrayList expectedNodes3 = new ArrayList<>(); + expectedNodes3.add(new SWHID("swh:1:rev:0000000000000000000000000000000000000013")); + GraphTest.assertEqualsAnyOrder(expectedNodes3, actuals3); + + TraversalRequest request4 = getNeighborsRequestBuilder(fakeSWHID("rev", 9)) + .setDirection(GraphDirection.BACKWARD).setEdges("rev:rev").build(); + ArrayList actuals4 = getSWHIDs(client.traverse(request4)); + ArrayList expectedNodes4 = new ArrayList<>(); + expectedNodes4.add(new SWHID("swh:1:rev:0000000000000000000000000000000000000013")); + GraphTest.assertEqualsAnyOrder(expectedNodes4, actuals4); + + TraversalRequest request5 = getNeighborsRequestBuilder(fakeSWHID("snp", 20)) + .setDirection(GraphDirection.BACKWARD).build(); + ArrayList actuals5 = getSWHIDs(client.traverse(request5)); + ArrayList expectedNodes5 = new ArrayList<>(); + expectedNodes5.add(new SWHID(TEST_ORIGIN_ID)); + GraphTest.assertEqualsAnyOrder(expectedNodes5, actuals5); + } + + @Test + public void twoNeighbors() { + TraversalRequest request1 = getNeighborsRequestBuilder(fakeSWHID("snp", 20)).build(); + ArrayList actuals1 = getSWHIDs(client.traverse(request1)); + ArrayList expectedNodes1 = new ArrayList<>(); + expectedNodes1.add(new SWHID("swh:1:rel:0000000000000000000000000000000000000010")); + expectedNodes1.add(new SWHID("swh:1:rev:0000000000000000000000000000000000000009")); + GraphTest.assertEqualsAnyOrder(expectedNodes1, actuals1); + + TraversalRequest request2 = getNeighborsRequestBuilder(fakeSWHID("dir", 8)).setEdges("dir:cnt").build(); + ArrayList actuals2 = getSWHIDs(client.traverse(request2)); + ArrayList expectedNodes2 = new ArrayList<>(); + expectedNodes2.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000001")); + expectedNodes2.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000007")); + GraphTest.assertEqualsAnyOrder(expectedNodes2, actuals2); + + TraversalRequest request3 = getNeighborsRequestBuilder(fakeSWHID("cnt", 1)) + .setDirection(GraphDirection.BACKWARD).build(); + ArrayList actuals3 = getSWHIDs(client.traverse(request3)); + ArrayList expectedNodes3 = new ArrayList<>(); + expectedNodes3.add(new SWHID("swh:1:dir:0000000000000000000000000000000000000008")); + expectedNodes3.add(new SWHID("swh:1:dir:0000000000000000000000000000000000000002")); + GraphTest.assertEqualsAnyOrder(expectedNodes3, actuals3); + + TraversalRequest request4 = getNeighborsRequestBuilder(fakeSWHID("rev", 9)) + .setDirection(GraphDirection.BACKWARD).setEdges("rev:snp,rev:rel").build(); + ArrayList actuals4 = getSWHIDs(client.traverse(request4)); + ArrayList expectedNodes4 = new ArrayList<>(); + expectedNodes4.add(new SWHID("swh:1:snp:0000000000000000000000000000000000000020")); + expectedNodes4.add(new SWHID("swh:1:rel:0000000000000000000000000000000000000010")); + GraphTest.assertEqualsAnyOrder(expectedNodes4, actuals4); + } + + @Test + public void threeNeighbors() { + TraversalRequest request1 = getNeighborsRequestBuilder(fakeSWHID("dir", 8)).build(); + ArrayList actuals1 = getSWHIDs(client.traverse(request1)); + ArrayList expectedNodes1 = new ArrayList<>(); + expectedNodes1.add(new SWHID("swh:1:dir:0000000000000000000000000000000000000006")); + expectedNodes1.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000001")); + expectedNodes1.add(new SWHID("swh:1:cnt:0000000000000000000000000000000000000007")); + GraphTest.assertEqualsAnyOrder(expectedNodes1, actuals1); + + TraversalRequest request2 = getNeighborsRequestBuilder(fakeSWHID("rev", 9)) + .setDirection(GraphDirection.BACKWARD).build(); + ArrayList actuals2 = getSWHIDs(client.traverse(request2)); + ArrayList expectedNodes2 = new ArrayList<>(); + expectedNodes2.add(new SWHID("swh:1:snp:0000000000000000000000000000000000000020")); + expectedNodes2.add(new SWHID("swh:1:rel:0000000000000000000000000000000000000010")); + expectedNodes2.add(new SWHID("swh:1:rev:0000000000000000000000000000000000000013")); + GraphTest.assertEqualsAnyOrder(expectedNodes2, actuals2); + } +} diff --git a/java/src/test/java/org/softwareheritage/graph/rpc/TraverseNodesTest.java b/java/src/test/java/org/softwareheritage/graph/rpc/TraverseNodesTest.java new file mode 100644 --- /dev/null +++ b/java/src/test/java/org/softwareheritage/graph/rpc/TraverseNodesTest.java @@ -0,0 +1,149 @@ +package org.softwareheritage.graph.rpc; + +import org.junit.jupiter.api.Test; +import org.softwareheritage.graph.GraphTest; +import org.softwareheritage.graph.SWHID; + +import java.util.ArrayList; +import java.util.List; + +public class TraverseNodesTest extends TraversalServiceTest { + private TraversalRequest.Builder getTraversalRequestBuilder(SWHID src) { + return TraversalRequest.newBuilder().addSrc(src.toString()); + } + + @Test + public void forwardFromRoot() { + ArrayList actual = getSWHIDs( + client.traverse(getTraversalRequestBuilder(new SWHID(TEST_ORIGIN_ID)).build())); + List expected = List.of(fakeSWHID("cnt", 1), fakeSWHID("cnt", 4), fakeSWHID("cnt", 5), + fakeSWHID("cnt", 7), fakeSWHID("dir", 2), fakeSWHID("dir", 6), fakeSWHID("dir", 8), + fakeSWHID("rel", 10), fakeSWHID("rev", 3), fakeSWHID("rev", 9), fakeSWHID("snp", 20), + new SWHID(TEST_ORIGIN_ID)); + GraphTest.assertEqualsAnyOrder(expected, actual); + } + + @Test + public void forwardFromMiddle() { + ArrayList actual = getSWHIDs(client.traverse(getTraversalRequestBuilder(fakeSWHID("dir", 12)).build())); + List expected = List.of(fakeSWHID("cnt", 1), fakeSWHID("cnt", 4), fakeSWHID("cnt", 5), + fakeSWHID("cnt", 7), fakeSWHID("cnt", 11), fakeSWHID("dir", 6), fakeSWHID("dir", 8), + fakeSWHID("dir", 12)); + GraphTest.assertEqualsAnyOrder(expected, actual); + } + + @Test + public void forwardRelRev() { + ArrayList actual = getSWHIDs( + client.traverse(getTraversalRequestBuilder(fakeSWHID("rel", 10)).setEdges("rel:rev,rev:rev").build())); + List expected = List.of(fakeSWHID("rel", 10), fakeSWHID("rev", 9), fakeSWHID("rev", 3)); + GraphTest.assertEqualsAnyOrder(expected, actual); + } + + @Test + public void forwardFilterReturnedNodesDir() { + ArrayList actual = getSWHIDs(client.traverse(getTraversalRequestBuilder(fakeSWHID("rel", 10)) + .setReturnNodes(NodeFilter.newBuilder().setTypes("dir").build()).build())); + List expected = List.of(fakeSWHID("dir", 2), fakeSWHID("dir", 8), fakeSWHID("dir", 6)); + GraphTest.assertEqualsAnyOrder(expected, actual); + } + + @Test + public void backwardFromRoot() { + ArrayList actual = getSWHIDs(client.traverse( + getTraversalRequestBuilder(new SWHID(TEST_ORIGIN_ID)).setDirection(GraphDirection.BACKWARD).build())); + List expected = List.of(new SWHID(TEST_ORIGIN_ID)); + GraphTest.assertEqualsAnyOrder(expected, actual); + } + + @Test + public void backwardFromMiddle() { + ArrayList actual = getSWHIDs(client.traverse( + getTraversalRequestBuilder(fakeSWHID("dir", 12)).setDirection(GraphDirection.BACKWARD).build())); + List expected = List.of(fakeSWHID("dir", 12), fakeSWHID("rel", 19), fakeSWHID("rev", 13), + fakeSWHID("rev", 18)); + GraphTest.assertEqualsAnyOrder(expected, actual); + } + + @Test + public void backwardFromLeaf() { + ArrayList actual = getSWHIDs(client.traverse( + getTraversalRequestBuilder(fakeSWHID("cnt", 4)).setDirection(GraphDirection.BACKWARD).build())); + List expected = List.of(new SWHID(TEST_ORIGIN_ID), fakeSWHID("cnt", 4), fakeSWHID("dir", 6), + fakeSWHID("dir", 8), fakeSWHID("dir", 12), fakeSWHID("rel", 10), fakeSWHID("rel", 19), + fakeSWHID("rev", 9), fakeSWHID("rev", 13), fakeSWHID("rev", 18), fakeSWHID("snp", 20)); + GraphTest.assertEqualsAnyOrder(expected, actual); + } + + @Test + public void forwardSnpToRev() { + ArrayList actual = getSWHIDs( + client.traverse(getTraversalRequestBuilder(fakeSWHID("snp", 20)).setEdges("snp:rev").build())); + List expected = List.of(fakeSWHID("rev", 9), fakeSWHID("snp", 20)); + GraphTest.assertEqualsAnyOrder(expected, actual); + } + + @Test + public void forwardRelToRevRevToRev() { + ArrayList actual = getSWHIDs( + client.traverse(getTraversalRequestBuilder(fakeSWHID("rel", 10)).setEdges("rel:rev,rev:rev").build())); + List expected = List.of(fakeSWHID("rel", 10), fakeSWHID("rev", 3), fakeSWHID("rev", 9)); + GraphTest.assertEqualsAnyOrder(expected, actual); + } + + @Test + public void forwardRevToAllDirToAll() { + ArrayList actual = getSWHIDs( + client.traverse(getTraversalRequestBuilder(fakeSWHID("rev", 13)).setEdges("rev:*,dir:*").build())); + List expected = List.of(fakeSWHID("cnt", 1), fakeSWHID("cnt", 4), fakeSWHID("cnt", 5), + fakeSWHID("cnt", 7), fakeSWHID("cnt", 11), fakeSWHID("dir", 2), fakeSWHID("dir", 6), + fakeSWHID("dir", 8), fakeSWHID("dir", 12), fakeSWHID("rev", 3), fakeSWHID("rev", 9), + fakeSWHID("rev", 13)); + GraphTest.assertEqualsAnyOrder(expected, actual); + } + + @Test + public void forwardSnpToAllRevToAll() { + ArrayList actual = getSWHIDs( + client.traverse(getTraversalRequestBuilder(fakeSWHID("snp", 20)).setEdges("snp:*,rev:*").build())); + List expected = List.of(fakeSWHID("dir", 2), fakeSWHID("dir", 8), fakeSWHID("rel", 10), + fakeSWHID("rev", 3), fakeSWHID("rev", 9), fakeSWHID("snp", 20)); + GraphTest.assertEqualsAnyOrder(expected, actual); + } + + @Test + public void forwardNoEdges() { + ArrayList actual = getSWHIDs( + client.traverse(getTraversalRequestBuilder(fakeSWHID("snp", 20)).setEdges("").build())); + List expected = List.of(fakeSWHID("snp", 20)); + GraphTest.assertEqualsAnyOrder(expected, actual); + } + + @Test + public void backwardRevToRevRevToRel() { + ArrayList actual = getSWHIDs(client.traverse(getTraversalRequestBuilder(fakeSWHID("rev", 3)) + .setEdges("rev:rev,rev:rel").setDirection(GraphDirection.BACKWARD).build())); + List expected = List.of(fakeSWHID("rel", 10), fakeSWHID("rel", 19), fakeSWHID("rev", 3), + fakeSWHID("rev", 9), fakeSWHID("rev", 13), fakeSWHID("rev", 18)); + GraphTest.assertEqualsAnyOrder(expected, actual); + } + + @Test + public void forwardFromRootNodesOnly() { + ArrayList actual = getSWHIDs( + client.traverse(getTraversalRequestBuilder(new SWHID(TEST_ORIGIN_ID)).build())); + List expected = List.of(new SWHID(TEST_ORIGIN_ID), fakeSWHID("cnt", 1), fakeSWHID("cnt", 4), + fakeSWHID("cnt", 5), fakeSWHID("cnt", 7), fakeSWHID("dir", 2), fakeSWHID("dir", 6), fakeSWHID("dir", 8), + fakeSWHID("rel", 10), fakeSWHID("rev", 3), fakeSWHID("rev", 9), fakeSWHID("snp", 20)); + GraphTest.assertEqualsAnyOrder(expected, actual); + } + + @Test + public void backwardRevToAllNodesOnly() { + ArrayList actual = getSWHIDs(client.traverse(getTraversalRequestBuilder(fakeSWHID("rev", 3)) + .setDirection(GraphDirection.BACKWARD).setEdges("rev:*").build())); + List expected = List.of(fakeSWHID("rel", 10), fakeSWHID("rel", 19), fakeSWHID("rev", 3), + fakeSWHID("rev", 9), fakeSWHID("rev", 13), fakeSWHID("rev", 18), fakeSWHID("snp", 20)); + GraphTest.assertEqualsAnyOrder(expected, actual); + } +} diff --git a/java/src/test/java/org/softwareheritage/graph/utils/ForkJoinBigQuickSort2Test.java b/java/src/test/java/org/softwareheritage/graph/utils/ForkJoinBigQuickSort2Test.java --- a/java/src/test/java/org/softwareheritage/graph/utils/ForkJoinBigQuickSort2Test.java +++ b/java/src/test/java/org/softwareheritage/graph/utils/ForkJoinBigQuickSort2Test.java @@ -82,15 +82,5 @@ d[1][i] = random.nextInt(); sortBig2(d[0], d[1], 10, 100); checkArraySorted(d[0], d[1], 10, 100); - - d[0] = new long[10000000]; - random = new Random(0); - for (int i = d[0].length; i-- != 0;) - d[0][i] = random.nextInt(); - d[1] = new long[d[0].length]; - for (int i = d[1].length; i-- != 0;) - d[1][i] = random.nextInt(); - sortBig2(d[0], d[1]); - checkArraySorted(d[0], d[1]); } } diff --git a/java/src/test/java/org/softwareheritage/graph/utils/ForkJoinQuickSort3Test.java b/java/src/test/java/org/softwareheritage/graph/utils/ForkJoinQuickSort3Test.java --- a/java/src/test/java/org/softwareheritage/graph/utils/ForkJoinQuickSort3Test.java +++ b/java/src/test/java/org/softwareheritage/graph/utils/ForkJoinQuickSort3Test.java @@ -86,18 +86,5 @@ d[2][i] = random.nextInt(); ForkJoinQuickSort3.parallelQuickSort(d[0], d[1], d[2], 10, 100); checkArraySorted(d[0], d[1], d[2], 10, 100); - - d[0] = new long[10000000]; - random = new Random(0); - for (int i = d[0].length; i-- != 0;) - d[0][i] = random.nextInt(); - d[1] = new long[d[0].length]; - for (int i = d[1].length; i-- != 0;) - d[1][i] = random.nextInt(); - d[2] = new long[d[0].length]; - for (int i = d[2].length; i-- != 0;) - d[2][i] = random.nextInt(); - ForkJoinQuickSort3.parallelQuickSort(d[0], d[1], d[2]); - checkArraySorted(d[0], d[1], d[2]); } } diff --git a/proto/swhgraph.proto b/proto/swhgraph.proto new file mode 100644 --- /dev/null +++ b/proto/swhgraph.proto @@ -0,0 +1,134 @@ +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "org.softwareheritage.graph.rpc"; +option java_outer_classname = "GraphService"; + +package swh.graph; + +service TraversalService { + rpc Traverse (TraversalRequest) returns (stream Node); + rpc CountNodes (TraversalRequest) returns (CountResponse); + rpc CountEdges (TraversalRequest) returns (CountResponse); + rpc Stats (StatsRequest) returns (StatsResponse); + rpc CheckSwhid (CheckSwhidRequest) returns (CheckSwhidResponse); +} + +enum GraphDirection { + FORWARD = 0; + BACKWARD = 1; + BOTH = 2; +} + +message TraversalRequest { + repeated string src = 1; + + // Traversal options + optional GraphDirection direction = 2; + optional string edges = 3; + optional int64 max_edges = 4; + optional int64 min_depth = 5; + optional int64 max_depth = 6; + optional NodeFilter return_nodes = 7; + optional NodeFields return_fields = 8; +} + +message NodeFilter { + optional string types = 1; + optional int64 min_traversal_successors = 2; + optional int64 max_traversal_successors = 3; +} + +message NodeFields { + optional bool swhid = 1; + + optional bool successor = 2; + optional bool successor_swhid = 3; + optional bool successor_label = 4; + + optional bool cnt_length = 5; + optional bool cnt_is_skipped = 6; + + optional bool rev_author = 7; + optional bool rev_author_date = 8; + optional bool rev_author_date_offset = 9; + optional bool rev_committer = 10; + optional bool rev_committer_date = 11; + optional bool rev_committer_date_offset = 12; + optional bool rev_message = 13; + + optional bool rel_author = 14; + optional bool rel_author_date = 15; + optional bool rel_author_date_offset = 16; + optional bool rel_name = 17; + optional bool rel_message = 18; + + optional bool ori_url = 19; +} + +message Node { + string swhid = 1; + repeated Successor successor = 2; + + optional int64 cnt_length = 3; + optional bool cnt_is_skipped = 4; + + optional int64 rev_author = 5; + optional int64 rev_author_date = 6; + optional int32 rev_author_date_offset = 7; + optional int64 rev_committer = 8; + optional int64 rev_committer_date = 9; + optional int32 rev_committer_date_offset = 10; + optional bytes rev_message = 11; + + optional int64 rel_author = 12; + optional int64 rel_author_date = 13; + optional int32 rel_author_date_offset = 14; + optional bytes rel_name = 15; + optional bytes rel_message = 16; + + optional string ori_url = 17; +} + +message Successor { + optional string swhid = 1; + repeated EdgeLabel label = 2; +} + +message EdgeLabel { + bytes name = 1; + int32 permission = 2; +} + +message CountResponse { + int64 count = 1; +} + +message StatsRequest { +} + +message StatsResponse { + int64 num_nodes = 1; + int64 num_edges = 2; + + double compression = 3; + double bits_per_node = 4; + double bits_per_edge = 5; + double avg_locality = 6; + + int64 indegree_min = 7; + int64 indegree_max = 8; + double indegree_avg = 9; + int64 outdegree_min = 10; + int64 outdegree_max = 11; + double outdegree_avg = 12; +} + +message CheckSwhidRequest { + string swhid = 1; +} + +message CheckSwhidResponse { + bool exists = 1; + string details = 2; +} diff --git a/pyproject.toml b/pyproject.toml --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,10 @@ [tool.black] target-version = ['py37'] +extend-exclude = ''' +/( + | swh/graph/rpc +)/ +''' [tool.isort] multi_line_output = 3 diff --git a/requirements-test.txt b/requirements-test.txt --- a/requirements-test.txt +++ b/requirements-test.txt @@ -4,3 +4,4 @@ types-click types-pyyaml types-requests +types-protobuf diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ click py4j psutil +grpcio-tools +mypy-protobuf diff --git a/setup.cfg b/setup.cfg --- a/setup.cfg +++ b/setup.cfg @@ -6,3 +6,4 @@ select = C,E,F,W,B950 ignore = E203,E231,E501,W503 max-line-length = 88 +exclude = swh/graph/rpc diff --git a/swh/graph/backend.py b/swh/graph/backend.py deleted file mode 100644 --- a/swh/graph/backend.py +++ /dev/null @@ -1,176 +0,0 @@ -# Copyright (C) 2019-2020 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import asyncio -import contextlib -import io -import os -import re -import subprocess -import sys -import tempfile - -from py4j.java_gateway import JavaGateway -from py4j.protocol import Py4JJavaError - -from swh.graph.config import check_config - -BUF_LINES = 1024 - - -def _get_pipe_stderr(): - # Get stderr if possible, or pipe to stdout if running with Jupyter. - try: - sys.stderr.fileno() - except io.UnsupportedOperation: - return subprocess.STDOUT - else: - return sys.stderr - - -class Backend: - def __init__(self, graph_path, config=None): - self.gateway = None - self.entry = None - self.graph_path = graph_path - self.config = check_config(config or {}) - - def start_gateway(self): - self.gateway = JavaGateway.launch_gateway( - java_path=None, - javaopts=self.config["java_tool_options"].split(), - classpath=self.config["classpath"], - die_on_exit=True, - redirect_stdout=sys.stdout, - redirect_stderr=_get_pipe_stderr(), - ) - self.entry = self.gateway.jvm.org.softwareheritage.graph.Entry() - self.entry.load_graph(self.graph_path) - self.stream_proxy = JavaStreamProxy(self.entry) - - def stop_gateway(self): - self.gateway.shutdown() - - def __enter__(self): - self.start_gateway() - return self - - def __exit__(self, exc_type, exc_value, tb): - self.stop_gateway() - - def stats(self): - return self.entry.stats() - - def check_swhid(self, swhid): - try: - self.entry.check_swhid(swhid) - except Py4JJavaError as e: - m = re.search(r"malformed SWHID: (\w+)", str(e)) - if m: - raise ValueError(f"malformed SWHID: {m[1]}") - m = re.search(r"Unknown SWHID: ([:\w]+)", str(e)) - if m: - raise NameError(f"Unknown SWHID: {m[1]}") - raise - - def count(self, ttype, *args): - method = getattr(self.entry, "count_" + ttype) - return method(*args) - - async def traversal(self, ttype, *args): - method = getattr(self.stream_proxy, ttype) - async for line in method(*args): - yield line.decode().rstrip("\n") - - -class JavaStreamProxy: - """A proxy class for the org.softwareheritage.graph.Entry Java class that - takes care of the setup and teardown of the named-pipe FIFO communication - between Python and Java. - - Initialize JavaStreamProxy using: - - proxy = JavaStreamProxy(swh_entry_class_instance) - - Then you can call an Entry method and iterate on the FIFO results like - this: - - async for value in proxy.java_method(arg1, arg2): - print(value) - """ - - def __init__(self, entry): - self.entry = entry - - async def read_node_ids(self, fname): - loop = asyncio.get_event_loop() - open_thread = loop.run_in_executor(None, open, fname, "rb") - - # Since the open() call on the FIFO is blocking until it is also opened - # on the Java side, we await it with a timeout in case there is an - # exception that prevents the write-side open(). - with (await asyncio.wait_for(open_thread, timeout=2)) as f: - - def read_n_lines(f, n): - buf = [] - for _ in range(n): - try: - buf.append(next(f)) - except StopIteration: - break - return buf - - while True: - lines = await loop.run_in_executor(None, read_n_lines, f, BUF_LINES) - if not lines: - break - for line in lines: - yield line - - class _HandlerWrapper: - def __init__(self, handler): - self._handler = handler - - def __getattr__(self, name): - func = getattr(self._handler, name) - - async def java_call(*args, **kwargs): - loop = asyncio.get_event_loop() - await loop.run_in_executor(None, lambda: func(*args, **kwargs)) - - def java_task(*args, **kwargs): - return asyncio.create_task(java_call(*args, **kwargs)) - - return java_task - - @contextlib.contextmanager - def get_handler(self): - with tempfile.TemporaryDirectory(prefix="swh-graph-") as tmpdirname: - cli_fifo = os.path.join(tmpdirname, "swh-graph.fifo") - os.mkfifo(cli_fifo) - reader = self.read_node_ids(cli_fifo) - query_handler = self.entry.get_handler(cli_fifo) - handler = self._HandlerWrapper(query_handler) - yield (handler, reader) - - def __getattr__(self, name): - async def java_call_iterator(*args, **kwargs): - with self.get_handler() as (handler, reader): - java_task = getattr(handler, name)(*args, **kwargs) - try: - async for value in reader: - yield value - except asyncio.TimeoutError: - # If the read-side open() timeouts, an exception on the - # Java side probably happened that prevented the - # write-side open(). We propagate this exception here if - # that is the case. - task_exc = java_task.exception() - if task_exc: - raise task_exc - raise - await java_task - - return java_call_iterator diff --git a/swh/graph/cli.py b/swh/graph/cli.py --- a/swh/graph/cli.py +++ b/swh/graph/cli.py @@ -122,9 +122,9 @@ @click.pass_context def serve(ctx, host, port, graph): """run the graph RPC service""" - import aiohttp + import aiohttp.web - from swh.graph.server.app import make_app + from swh.graph.http_server import make_app config = ctx.obj["config"] config.setdefault("graph", {}) diff --git a/swh/graph/dot.py b/swh/graph/dot.py deleted file mode 100644 --- a/swh/graph/dot.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (C) 2019 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import collections -from functools import lru_cache -import subprocess - -KIND_TO_SHAPE = { - "ori": "egg", - "snp": "doubleoctagon", - "rel": "octagon", - "rev": "diamond", - "dir": "folder", - "cnt": "oval", -} - - -@lru_cache() -def dot_to_svg(dot): - try: - p = subprocess.run( - ["dot", "-Tsvg"], - input=dot, - universal_newlines=True, - capture_output=True, - check=True, - ) - except subprocess.CalledProcessError as e: - raise RuntimeError(e.stderr) from e - return p.stdout - - -def graph_dot(nodes): - ids = {n.id for n in nodes} - - by_kind = collections.defaultdict(list) - for n in nodes: - by_kind[n.kind].append(n) - - forward_edges = [ - (node.id, child.id) - for node in nodes - for child in node.children() - if child.id in ids - ] - backward_edges = [ - (parent.id, node.id) - for node in nodes - for parent in node.parents() - if parent.id in ids - ] - edges = set(forward_edges + backward_edges) - edges_fmt = "\n".join("{} -> {};".format(a, b) for a, b in edges) - nodes_fmt = "\n".join(node.dot_fragment() for node in nodes) - - s = """digraph G {{ - ranksep=1; - nodesep=0.5; - - {nodes} - {edges} - - }}""".format( - nodes=nodes_fmt, edges=edges_fmt - ) - return s diff --git a/swh/graph/client.py b/swh/graph/http_client.py rename from swh/graph/client.py rename to swh/graph/http_client.py diff --git a/swh/graph/naive_client.py b/swh/graph/http_naive_client.py rename from swh/graph/naive_client.py rename to swh/graph/http_naive_client.py --- a/swh/graph/naive_client.py +++ b/swh/graph/http_naive_client.py @@ -22,7 +22,7 @@ from swh.model.swhids import CoreSWHID, ExtendedSWHID, ValidationError -from .client import GraphArgumentException +from .http_client import GraphArgumentException _NODE_TYPES = "ori|snp|rel|rev|dir|cnt" NODES_RE = re.compile(rf"(\*|{_NODE_TYPES})") @@ -81,10 +81,10 @@ class NaiveClient: - """An alternative implementation of :class:`swh.graph.backend.Backend`, - written in pure-python and meant for simulating it in other components' test - cases; constructed from a list of nodes and (directed) edges, both - represented as SWHIDs. + """An alternative implementation of the graph server, written in + pure-python and meant for simulating it in other components' test cases; + constructed from a list of nodes and (directed) edges, both represented as + SWHIDs. It is NOT meant to be efficient in any way; only to be a very simple implementation that provides the same behavior. @@ -124,26 +124,22 @@ def stats(self) -> Dict: return { - "counts": { - "nodes": len(self.graph.nodes), - "edges": sum(map(len, self.graph.forward_edges.values())), - }, - "ratios": { - "compression": 1.0, - "bits_per_edge": 100.0, - "bits_per_node": 100.0, - "avg_locality": 0.0, - }, - "indegree": { - "min": min(map(len, self.graph.backward_edges.values())), - "max": max(map(len, self.graph.backward_edges.values())), - "avg": statistics.mean(map(len, self.graph.backward_edges.values())), - }, - "outdegree": { - "min": min(map(len, self.graph.forward_edges.values())), - "max": max(map(len, self.graph.forward_edges.values())), - "avg": statistics.mean(map(len, self.graph.forward_edges.values())), - }, + "num_nodes": len(self.graph.nodes), + "num_edges": sum(map(len, self.graph.forward_edges.values())), + "compression": 1.0, + "bits_per_edge": 100.0, + "bits_per_node": 100.0, + "avg_locality": 0.0, + "indegree_min": min(map(len, self.graph.backward_edges.values())), + "indegree_max": max(map(len, self.graph.backward_edges.values())), + "indegree_avg": statistics.mean( + map(len, self.graph.backward_edges.values()) + ), + "outdegree_min": min(map(len, self.graph.forward_edges.values())), + "outdegree_max": max(map(len, self.graph.forward_edges.values())), + "outdegree_avg": statistics.mean( + map(len, self.graph.forward_edges.values()) + ), } @check_arguments diff --git a/swh/graph/server/app.py b/swh/graph/http_server.py rename from swh/graph/server/app.py rename to swh/graph/http_server.py --- a/swh/graph/server/app.py +++ b/swh/graph/http_server.py @@ -8,16 +8,26 @@ FIFO as a transport to stream integers between the two languages. """ -import asyncio -from collections import deque +import json import os from typing import Optional +import aiohttp.test_utils import aiohttp.web +from google.protobuf import json_format +import grpc from swh.core.api.asynchronous import RPCServerApp from swh.core.config import read as config_read -from swh.graph.backend import Backend +from swh.graph.rpc.swhgraph_pb2 import ( + CheckSwhidRequest, + NodeFields, + NodeFilter, + StatsRequest, + TraversalRequest, +) +from swh.graph.rpc.swhgraph_pb2_grpc import TraversalServiceStub +from swh.graph.rpc_server import spawn_java_rpc_server from swh.model.swhids import EXTENDED_SWHID_TYPES try: @@ -34,18 +44,21 @@ class GraphServerApp(RPCServerApp): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.on_startup.append(self._start_gateway) - self.on_shutdown.append(self._stop_gateway) + self.on_startup.append(self._start) + self.on_shutdown.append(self._stop) @staticmethod - async def _start_gateway(app): - # Equivalent to entering `with app["backend"]:` - app["backend"].start_gateway() + async def _start(app): + app["channel"] = grpc.aio.insecure_channel(app["rpc_url"]) + await app["channel"].__aenter__() + app["rpc_client"] = TraversalServiceStub(app["channel"]) + await app["rpc_client"].Stats(StatsRequest(), wait_for_ready=True) @staticmethod - async def _stop_gateway(app): - # Equivalent to exiting `with app["backend"]:` with no error - app["backend"].stop_gateway() + async def _stop(app): + await app["channel"].__aexit__(None, None, None) + if app.get("local_server"): + app["local_server"].terminate() async def index(request): @@ -70,14 +83,14 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.backend = self.request.app["backend"] + self.rpc_client: TraversalServiceStub = self.request.app["rpc_client"] def get_direction(self): """Validate HTTP query parameter `direction`""" s = self.request.query.get("direction", "forward") if s not in ("forward", "backward"): raise aiohttp.web.HTTPBadRequest(text=f"invalid direction: {s}") - return s + return s.upper() def get_edges(self): """Validate HTTP query parameter `edges`, i.e., edge restrictions""" @@ -134,12 +147,11 @@ except ValueError: raise aiohttp.web.HTTPBadRequest(text=f"invalid max_edges value: {s}") - def check_swhid(self, swhid): + async def check_swhid(self, swhid): """Validate that the given SWHID exists in the graph""" - try: - self.backend.check_swhid(swhid) - except (NameError, ValueError) as e: - raise aiohttp.web.HTTPBadRequest(text=str(e)) + r = await self.rpc_client.CheckSwhid(CheckSwhidRequest(swhid=swhid)) + if not r.exists: + raise aiohttp.web.HTTPBadRequest(text=str(r.details)) class StreamingGraphView(GraphView): @@ -193,109 +205,69 @@ """View showing some statistics on the graph""" async def get(self): - stats = self.backend.stats() - return aiohttp.web.Response(body=stats, content_type="application/json") + res = await self.rpc_client.Stats(StatsRequest()) + stats = json_format.MessageToDict( + res, including_default_value_fields=True, preserving_proto_field_name=True + ) + # Int64 fields are serialized as strings by default. + for descriptor in res.DESCRIPTOR.fields: + if descriptor.type == descriptor.TYPE_INT64: + try: + stats[descriptor.name] = int(stats[descriptor.name]) + except KeyError: + pass + json_body = json.dumps(stats, indent=4, sort_keys=True) + return aiohttp.web.Response(body=json_body, content_type="application/json") class SimpleTraversalView(StreamingGraphView): """Base class for views of simple traversals""" - simple_traversal_type: Optional[str] = None - async def prepare_response(self): - self.src = self.request.match_info["src"] - self.edges = self.get_edges() - self.direction = self.get_direction() - self.max_edges = self.get_max_edges() - self.return_types = self.get_return_types() - self.check_swhid(self.src) + src = self.request.match_info["src"] + self.traversal_request = TraversalRequest( + src=[src], + edges=self.get_edges(), + direction=self.get_direction(), + return_nodes=NodeFilter(types=self.get_return_types()), + return_fields=NodeFields(), + ) + if self.get_max_edges(): + self.traversal_request.max_edges = self.get_max_edges() + await self.check_swhid(src) + self.configure_request() + + def configure_request(self): + pass async def stream_response(self): - async for res_line in self.backend.traversal( - self.simple_traversal_type, - self.direction, - self.edges, - self.src, - self.max_edges, - self.return_types, - ): - await self.stream_line(res_line) + async for node in self.rpc_client.Traverse(self.traversal_request): + await self.stream_line(node.swhid) class LeavesView(SimpleTraversalView): - simple_traversal_type = "leaves" + def configure_request(self): + self.traversal_request.return_nodes.max_traversal_successors = 0 class NeighborsView(SimpleTraversalView): - simple_traversal_type = "neighbors" + def configure_request(self): + self.traversal_request.min_depth = 1 + self.traversal_request.max_depth = 1 class VisitNodesView(SimpleTraversalView): - simple_traversal_type = "visit_nodes" + pass class VisitEdgesView(SimpleTraversalView): - simple_traversal_type = "visit_edges" - - -class WalkView(StreamingGraphView): - async def prepare_response(self): - self.src = self.request.match_info["src"] - self.dst = self.request.match_info["dst"] - - self.edges = self.get_edges() - self.direction = self.get_direction() - self.algo = self.get_traversal() - self.limit = self.get_limit() - self.max_edges = self.get_max_edges() - self.return_types = self.get_return_types() - - self.check_swhid(self.src) - if self.dst not in EXTENDED_SWHID_TYPES: - self.check_swhid(self.dst) - - async def get_walk_iterator(self): - return self.backend.traversal( - "walk", - self.direction, - self.edges, - self.algo, - self.src, - self.dst, - self.max_edges, - self.return_types, - ) + def configure_request(self): + self.traversal_request.return_fields.successor = True async def stream_response(self): - it = self.get_walk_iterator() - if self.limit < 0: - queue = deque(maxlen=-self.limit) - async for res_swhid in it: - queue.append(res_swhid) - while queue: - await self.stream_line(queue.popleft()) - else: - count = 0 - async for res_swhid in it: - if self.limit == 0 or count < self.limit: - await self.stream_line(res_swhid) - count += 1 - else: - break - - -class RandomWalkView(WalkView): - def get_walk_iterator(self): - return self.backend.traversal( - "random_walk", - self.direction, - self.edges, - RANDOM_RETRIES, - self.src, - self.dst, - self.max_edges, - self.return_types, - ) + async for node in self.rpc_client.Traverse(self.traversal_request): + for succ in node.successor: + await self.stream_line(node.swhid + " " + succ.swhid) class CountView(GraphView): @@ -304,44 +276,48 @@ count_type: Optional[str] = None async def get(self): - self.src = self.request.match_info["src"] - self.check_swhid(self.src) - - self.edges = self.get_edges() - self.direction = self.get_direction() - self.max_edges = self.get_max_edges() - - loop = asyncio.get_event_loop() - cnt = await loop.run_in_executor( - None, - self.backend.count, - self.count_type, - self.direction, - self.edges, - self.src, - self.max_edges, + src = self.request.match_info["src"] + self.traversal_request = TraversalRequest( + src=[src], + edges=self.get_edges(), + direction=self.get_direction(), + return_nodes=NodeFilter(types=self.get_return_types()), + return_fields=NodeFields(), + ) + if self.get_max_edges(): + self.traversal_request.max_edges = self.get_max_edges() + self.configure_request() + res = await self.rpc_client.CountNodes(self.traversal_request) + return aiohttp.web.Response( + body=str(res.count), content_type="application/json" ) - return aiohttp.web.Response(body=str(cnt), content_type="application/json") + + def configure_request(self): + pass class CountNeighborsView(CountView): - count_type = "neighbors" + def configure_request(self): + self.traversal_request.min_depth = 1 + self.traversal_request.max_depth = 1 class CountLeavesView(CountView): - count_type = "leaves" + def configure_request(self): + self.traversal_request.return_nodes.max_traversal_successors = 0 class CountVisitNodesView(CountView): - count_type = "visit_nodes" + pass -def make_app(config=None, backend=None, **kwargs): - if (config is None) == (backend is None): - raise ValueError("make_app() expects exactly one of 'config' or 'backend'") - if backend is None: - backend = Backend(graph_path=config["graph"]["path"], config=config["graph"]) +def make_app(config=None, rpc_url=None, **kwargs): app = GraphServerApp(**kwargs) + + if rpc_url is None: + app["local_server"], port = spawn_java_rpc_server(config) + rpc_url = f"localhost:{port}" + app.add_routes( [ aiohttp.web.get("/", index), @@ -351,16 +327,13 @@ aiohttp.web.view("/graph/neighbors/{src}", NeighborsView), aiohttp.web.view("/graph/visit/nodes/{src}", VisitNodesView), aiohttp.web.view("/graph/visit/edges/{src}", VisitEdgesView), - # temporarily disabled in wait of a proper fix for T1969 - # aiohttp.web.view("/graph/walk/{src}/{dst}", WalkView) - aiohttp.web.view("/graph/randomwalk/{src}/{dst}", RandomWalkView), aiohttp.web.view("/graph/neighbors/count/{src}", CountNeighborsView), aiohttp.web.view("/graph/leaves/count/{src}", CountLeavesView), aiohttp.web.view("/graph/visit/nodes/count/{src}", CountVisitNodesView), ] ) - app["backend"] = backend + app["rpc_url"] = rpc_url return app diff --git a/swh/graph/rpc/swhgraph.proto b/swh/graph/rpc/swhgraph.proto new file mode 120000 --- /dev/null +++ b/swh/graph/rpc/swhgraph.proto @@ -0,0 +1 @@ +../../../proto/swhgraph.proto \ No newline at end of file diff --git a/swh/graph/rpc/swhgraph_pb2.py b/swh/graph/rpc/swhgraph_pb2.py new file mode 100644 --- /dev/null +++ b/swh/graph/rpc/swhgraph_pb2.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: swh/graph/rpc/swhgraph.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cswh/graph/rpc/swhgraph.proto\x12\tswh.graph\"\xf8\x02\n\x10TraversalRequest\x12\x0b\n\x03src\x18\x01 \x03(\t\x12\x31\n\tdirection\x18\x02 \x01(\x0e\x32\x19.swh.graph.GraphDirectionH\x00\x88\x01\x01\x12\x12\n\x05\x65\x64ges\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tmax_edges\x18\x04 \x01(\x03H\x02\x88\x01\x01\x12\x16\n\tmin_depth\x18\x05 \x01(\x03H\x03\x88\x01\x01\x12\x16\n\tmax_depth\x18\x06 \x01(\x03H\x04\x88\x01\x01\x12\x30\n\x0creturn_nodes\x18\x07 \x01(\x0b\x32\x15.swh.graph.NodeFilterH\x05\x88\x01\x01\x12\x31\n\rreturn_fields\x18\x08 \x01(\x0b\x32\x15.swh.graph.NodeFieldsH\x06\x88\x01\x01\x42\x0c\n\n_directionB\x08\n\x06_edgesB\x0c\n\n_max_edgesB\x0c\n\n_min_depthB\x0c\n\n_max_depthB\x0f\n\r_return_nodesB\x10\n\x0e_return_fields\"\xb2\x01\n\nNodeFilter\x12\x12\n\x05types\x18\x01 \x01(\tH\x00\x88\x01\x01\x12%\n\x18min_traversal_successors\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12%\n\x18max_traversal_successors\x18\x03 \x01(\x03H\x02\x88\x01\x01\x42\x08\n\x06_typesB\x1b\n\x19_min_traversal_successorsB\x1b\n\x19_max_traversal_successors\"\x86\x07\n\nNodeFields\x12\x12\n\x05swhid\x18\x01 \x01(\x08H\x00\x88\x01\x01\x12\x16\n\tsuccessor\x18\x02 \x01(\x08H\x01\x88\x01\x01\x12\x1c\n\x0fsuccessor_swhid\x18\x03 \x01(\x08H\x02\x88\x01\x01\x12\x1c\n\x0fsuccessor_label\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x17\n\ncnt_length\x18\x05 \x01(\x08H\x04\x88\x01\x01\x12\x1b\n\x0e\x63nt_is_skipped\x18\x06 \x01(\x08H\x05\x88\x01\x01\x12\x17\n\nrev_author\x18\x07 \x01(\x08H\x06\x88\x01\x01\x12\x1c\n\x0frev_author_date\x18\x08 \x01(\x08H\x07\x88\x01\x01\x12#\n\x16rev_author_date_offset\x18\t \x01(\x08H\x08\x88\x01\x01\x12\x1a\n\rrev_committer\x18\n \x01(\x08H\t\x88\x01\x01\x12\x1f\n\x12rev_committer_date\x18\x0b \x01(\x08H\n\x88\x01\x01\x12&\n\x19rev_committer_date_offset\x18\x0c \x01(\x08H\x0b\x88\x01\x01\x12\x18\n\x0brev_message\x18\r \x01(\x08H\x0c\x88\x01\x01\x12\x17\n\nrel_author\x18\x0e \x01(\x08H\r\x88\x01\x01\x12\x1c\n\x0frel_author_date\x18\x0f \x01(\x08H\x0e\x88\x01\x01\x12#\n\x16rel_author_date_offset\x18\x10 \x01(\x08H\x0f\x88\x01\x01\x12\x15\n\x08rel_name\x18\x11 \x01(\x08H\x10\x88\x01\x01\x12\x18\n\x0brel_message\x18\x12 \x01(\x08H\x11\x88\x01\x01\x12\x14\n\x07ori_url\x18\x13 \x01(\x08H\x12\x88\x01\x01\x42\x08\n\x06_swhidB\x0c\n\n_successorB\x12\n\x10_successor_swhidB\x12\n\x10_successor_labelB\r\n\x0b_cnt_lengthB\x11\n\x0f_cnt_is_skippedB\r\n\x0b_rev_authorB\x12\n\x10_rev_author_dateB\x19\n\x17_rev_author_date_offsetB\x10\n\x0e_rev_committerB\x15\n\x13_rev_committer_dateB\x1c\n\x1a_rev_committer_date_offsetB\x0e\n\x0c_rev_messageB\r\n\x0b_rel_authorB\x12\n\x10_rel_author_dateB\x19\n\x17_rel_author_date_offsetB\x0b\n\t_rel_nameB\x0e\n\x0c_rel_messageB\n\n\x08_ori_url\"\x90\x06\n\x04Node\x12\r\n\x05swhid\x18\x01 \x01(\t\x12\'\n\tsuccessor\x18\x02 \x03(\x0b\x32\x14.swh.graph.Successor\x12\x17\n\ncnt_length\x18\x03 \x01(\x03H\x00\x88\x01\x01\x12\x1b\n\x0e\x63nt_is_skipped\x18\x04 \x01(\x08H\x01\x88\x01\x01\x12\x17\n\nrev_author\x18\x05 \x01(\x03H\x02\x88\x01\x01\x12\x1c\n\x0frev_author_date\x18\x06 \x01(\x03H\x03\x88\x01\x01\x12#\n\x16rev_author_date_offset\x18\x07 \x01(\x05H\x04\x88\x01\x01\x12\x1a\n\rrev_committer\x18\x08 \x01(\x03H\x05\x88\x01\x01\x12\x1f\n\x12rev_committer_date\x18\t \x01(\x03H\x06\x88\x01\x01\x12&\n\x19rev_committer_date_offset\x18\n \x01(\x05H\x07\x88\x01\x01\x12\x18\n\x0brev_message\x18\x0b \x01(\x0cH\x08\x88\x01\x01\x12\x17\n\nrel_author\x18\x0c \x01(\x03H\t\x88\x01\x01\x12\x1c\n\x0frel_author_date\x18\r \x01(\x03H\n\x88\x01\x01\x12#\n\x16rel_author_date_offset\x18\x0e \x01(\x05H\x0b\x88\x01\x01\x12\x15\n\x08rel_name\x18\x0f \x01(\x0cH\x0c\x88\x01\x01\x12\x18\n\x0brel_message\x18\x10 \x01(\x0cH\r\x88\x01\x01\x12\x14\n\x07ori_url\x18\x11 \x01(\tH\x0e\x88\x01\x01\x42\r\n\x0b_cnt_lengthB\x11\n\x0f_cnt_is_skippedB\r\n\x0b_rev_authorB\x12\n\x10_rev_author_dateB\x19\n\x17_rev_author_date_offsetB\x10\n\x0e_rev_committerB\x15\n\x13_rev_committer_dateB\x1c\n\x1a_rev_committer_date_offsetB\x0e\n\x0c_rev_messageB\r\n\x0b_rel_authorB\x12\n\x10_rel_author_dateB\x19\n\x17_rel_author_date_offsetB\x0b\n\t_rel_nameB\x0e\n\x0c_rel_messageB\n\n\x08_ori_url\"N\n\tSuccessor\x12\x12\n\x05swhid\x18\x01 \x01(\tH\x00\x88\x01\x01\x12#\n\x05label\x18\x02 \x03(\x0b\x32\x14.swh.graph.EdgeLabelB\x08\n\x06_swhid\"-\n\tEdgeLabel\x12\x0c\n\x04name\x18\x01 \x01(\x0c\x12\x12\n\npermission\x18\x02 \x01(\x05\"\x1e\n\rCountResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\"\x0e\n\x0cStatsRequest\"\x95\x02\n\rStatsResponse\x12\x11\n\tnum_nodes\x18\x01 \x01(\x03\x12\x11\n\tnum_edges\x18\x02 \x01(\x03\x12\x13\n\x0b\x63ompression\x18\x03 \x01(\x01\x12\x15\n\rbits_per_node\x18\x04 \x01(\x01\x12\x15\n\rbits_per_edge\x18\x05 \x01(\x01\x12\x14\n\x0c\x61vg_locality\x18\x06 \x01(\x01\x12\x14\n\x0cindegree_min\x18\x07 \x01(\x03\x12\x14\n\x0cindegree_max\x18\x08 \x01(\x03\x12\x14\n\x0cindegree_avg\x18\t \x01(\x01\x12\x15\n\routdegree_min\x18\n \x01(\x03\x12\x15\n\routdegree_max\x18\x0b \x01(\x03\x12\x15\n\routdegree_avg\x18\x0c \x01(\x01\"\"\n\x11\x43heckSwhidRequest\x12\r\n\x05swhid\x18\x01 \x01(\t\"5\n\x12\x43heckSwhidResponse\x12\x0e\n\x06\x65xists\x18\x01 \x01(\x08\x12\x0f\n\x07\x64\x65tails\x18\x02 \x01(\t*5\n\x0eGraphDirection\x12\x0b\n\x07\x46ORWARD\x10\x00\x12\x0c\n\x08\x42\x41\x43KWARD\x10\x01\x12\x08\n\x04\x42OTH\x10\x02\x32\xdf\x02\n\x10TraversalService\x12:\n\x08Traverse\x12\x1b.swh.graph.TraversalRequest\x1a\x0f.swh.graph.Node0\x01\x12\x43\n\nCountNodes\x12\x1b.swh.graph.TraversalRequest\x1a\x18.swh.graph.CountResponse\x12\x43\n\nCountEdges\x12\x1b.swh.graph.TraversalRequest\x1a\x18.swh.graph.CountResponse\x12:\n\x05Stats\x12\x17.swh.graph.StatsRequest\x1a\x18.swh.graph.StatsResponse\x12I\n\nCheckSwhid\x12\x1c.swh.graph.CheckSwhidRequest\x1a\x1d.swh.graph.CheckSwhidResponseB0\n\x1eorg.softwareheritage.graph.rpcB\x0cGraphServiceP\x01\x62\x06proto3') + +_GRAPHDIRECTION = DESCRIPTOR.enum_types_by_name['GraphDirection'] +GraphDirection = enum_type_wrapper.EnumTypeWrapper(_GRAPHDIRECTION) +FORWARD = 0 +BACKWARD = 1 +BOTH = 2 + + +_TRAVERSALREQUEST = DESCRIPTOR.message_types_by_name['TraversalRequest'] +_NODEFILTER = DESCRIPTOR.message_types_by_name['NodeFilter'] +_NODEFIELDS = DESCRIPTOR.message_types_by_name['NodeFields'] +_NODE = DESCRIPTOR.message_types_by_name['Node'] +_SUCCESSOR = DESCRIPTOR.message_types_by_name['Successor'] +_EDGELABEL = DESCRIPTOR.message_types_by_name['EdgeLabel'] +_COUNTRESPONSE = DESCRIPTOR.message_types_by_name['CountResponse'] +_STATSREQUEST = DESCRIPTOR.message_types_by_name['StatsRequest'] +_STATSRESPONSE = DESCRIPTOR.message_types_by_name['StatsResponse'] +_CHECKSWHIDREQUEST = DESCRIPTOR.message_types_by_name['CheckSwhidRequest'] +_CHECKSWHIDRESPONSE = DESCRIPTOR.message_types_by_name['CheckSwhidResponse'] +TraversalRequest = _reflection.GeneratedProtocolMessageType('TraversalRequest', (_message.Message,), { + 'DESCRIPTOR' : _TRAVERSALREQUEST, + '__module__' : 'swh.graph.rpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.TraversalRequest) + }) +_sym_db.RegisterMessage(TraversalRequest) + +NodeFilter = _reflection.GeneratedProtocolMessageType('NodeFilter', (_message.Message,), { + 'DESCRIPTOR' : _NODEFILTER, + '__module__' : 'swh.graph.rpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.NodeFilter) + }) +_sym_db.RegisterMessage(NodeFilter) + +NodeFields = _reflection.GeneratedProtocolMessageType('NodeFields', (_message.Message,), { + 'DESCRIPTOR' : _NODEFIELDS, + '__module__' : 'swh.graph.rpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.NodeFields) + }) +_sym_db.RegisterMessage(NodeFields) + +Node = _reflection.GeneratedProtocolMessageType('Node', (_message.Message,), { + 'DESCRIPTOR' : _NODE, + '__module__' : 'swh.graph.rpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.Node) + }) +_sym_db.RegisterMessage(Node) + +Successor = _reflection.GeneratedProtocolMessageType('Successor', (_message.Message,), { + 'DESCRIPTOR' : _SUCCESSOR, + '__module__' : 'swh.graph.rpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.Successor) + }) +_sym_db.RegisterMessage(Successor) + +EdgeLabel = _reflection.GeneratedProtocolMessageType('EdgeLabel', (_message.Message,), { + 'DESCRIPTOR' : _EDGELABEL, + '__module__' : 'swh.graph.rpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.EdgeLabel) + }) +_sym_db.RegisterMessage(EdgeLabel) + +CountResponse = _reflection.GeneratedProtocolMessageType('CountResponse', (_message.Message,), { + 'DESCRIPTOR' : _COUNTRESPONSE, + '__module__' : 'swh.graph.rpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.CountResponse) + }) +_sym_db.RegisterMessage(CountResponse) + +StatsRequest = _reflection.GeneratedProtocolMessageType('StatsRequest', (_message.Message,), { + 'DESCRIPTOR' : _STATSREQUEST, + '__module__' : 'swh.graph.rpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.StatsRequest) + }) +_sym_db.RegisterMessage(StatsRequest) + +StatsResponse = _reflection.GeneratedProtocolMessageType('StatsResponse', (_message.Message,), { + 'DESCRIPTOR' : _STATSRESPONSE, + '__module__' : 'swh.graph.rpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.StatsResponse) + }) +_sym_db.RegisterMessage(StatsResponse) + +CheckSwhidRequest = _reflection.GeneratedProtocolMessageType('CheckSwhidRequest', (_message.Message,), { + 'DESCRIPTOR' : _CHECKSWHIDREQUEST, + '__module__' : 'swh.graph.rpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.CheckSwhidRequest) + }) +_sym_db.RegisterMessage(CheckSwhidRequest) + +CheckSwhidResponse = _reflection.GeneratedProtocolMessageType('CheckSwhidResponse', (_message.Message,), { + 'DESCRIPTOR' : _CHECKSWHIDRESPONSE, + '__module__' : 'swh.graph.rpc.swhgraph_pb2' + # @@protoc_insertion_point(class_scope:swh.graph.CheckSwhidResponse) + }) +_sym_db.RegisterMessage(CheckSwhidResponse) + +_TRAVERSALSERVICE = DESCRIPTOR.services_by_name['TraversalService'] +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\036org.softwareheritage.graph.rpcB\014GraphServiceP\001' + _GRAPHDIRECTION._serialized_start=2841 + _GRAPHDIRECTION._serialized_end=2894 + _TRAVERSALREQUEST._serialized_start=44 + _TRAVERSALREQUEST._serialized_end=420 + _NODEFILTER._serialized_start=423 + _NODEFILTER._serialized_end=601 + _NODEFIELDS._serialized_start=604 + _NODEFIELDS._serialized_end=1506 + _NODE._serialized_start=1509 + _NODE._serialized_end=2293 + _SUCCESSOR._serialized_start=2295 + _SUCCESSOR._serialized_end=2373 + _EDGELABEL._serialized_start=2375 + _EDGELABEL._serialized_end=2420 + _COUNTRESPONSE._serialized_start=2422 + _COUNTRESPONSE._serialized_end=2452 + _STATSREQUEST._serialized_start=2454 + _STATSREQUEST._serialized_end=2468 + _STATSRESPONSE._serialized_start=2471 + _STATSRESPONSE._serialized_end=2748 + _CHECKSWHIDREQUEST._serialized_start=2750 + _CHECKSWHIDREQUEST._serialized_end=2784 + _CHECKSWHIDRESPONSE._serialized_start=2786 + _CHECKSWHIDRESPONSE._serialized_end=2839 + _TRAVERSALSERVICE._serialized_start=2897 + _TRAVERSALSERVICE._serialized_end=3248 +# @@protoc_insertion_point(module_scope) diff --git a/swh/graph/rpc/swhgraph_pb2.pyi b/swh/graph/rpc/swhgraph_pb2.pyi new file mode 100644 --- /dev/null +++ b/swh/graph/rpc/swhgraph_pb2.pyi @@ -0,0 +1,418 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" +import builtins +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.internal.enum_type_wrapper +import google.protobuf.message +import typing +import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _GraphDirection: + ValueType = typing.NewType('ValueType', builtins.int) + V: typing_extensions.TypeAlias = ValueType +class _GraphDirectionEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_GraphDirection.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + FORWARD: _GraphDirection.ValueType # 0 + BACKWARD: _GraphDirection.ValueType # 1 + BOTH: _GraphDirection.ValueType # 2 +class GraphDirection(_GraphDirection, metaclass=_GraphDirectionEnumTypeWrapper): + pass + +FORWARD: GraphDirection.ValueType # 0 +BACKWARD: GraphDirection.ValueType # 1 +BOTH: GraphDirection.ValueType # 2 +global___GraphDirection = GraphDirection + + +class TraversalRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + SRC_FIELD_NUMBER: builtins.int + DIRECTION_FIELD_NUMBER: builtins.int + EDGES_FIELD_NUMBER: builtins.int + MAX_EDGES_FIELD_NUMBER: builtins.int + MIN_DEPTH_FIELD_NUMBER: builtins.int + MAX_DEPTH_FIELD_NUMBER: builtins.int + RETURN_NODES_FIELD_NUMBER: builtins.int + RETURN_FIELDS_FIELD_NUMBER: builtins.int + @property + def src(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[typing.Text]: ... + direction: global___GraphDirection.ValueType + """Traversal options""" + + edges: typing.Text + max_edges: builtins.int + min_depth: builtins.int + max_depth: builtins.int + @property + def return_nodes(self) -> global___NodeFilter: ... + @property + def return_fields(self) -> global___NodeFields: ... + def __init__(self, + *, + src: typing.Optional[typing.Iterable[typing.Text]] = ..., + direction: typing.Optional[global___GraphDirection.ValueType] = ..., + edges: typing.Optional[typing.Text] = ..., + max_edges: typing.Optional[builtins.int] = ..., + min_depth: typing.Optional[builtins.int] = ..., + max_depth: typing.Optional[builtins.int] = ..., + return_nodes: typing.Optional[global___NodeFilter] = ..., + return_fields: typing.Optional[global___NodeFields] = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_direction",b"_direction","_edges",b"_edges","_max_depth",b"_max_depth","_max_edges",b"_max_edges","_min_depth",b"_min_depth","_return_fields",b"_return_fields","_return_nodes",b"_return_nodes","direction",b"direction","edges",b"edges","max_depth",b"max_depth","max_edges",b"max_edges","min_depth",b"min_depth","return_fields",b"return_fields","return_nodes",b"return_nodes"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_direction",b"_direction","_edges",b"_edges","_max_depth",b"_max_depth","_max_edges",b"_max_edges","_min_depth",b"_min_depth","_return_fields",b"_return_fields","_return_nodes",b"_return_nodes","direction",b"direction","edges",b"edges","max_depth",b"max_depth","max_edges",b"max_edges","min_depth",b"min_depth","return_fields",b"return_fields","return_nodes",b"return_nodes","src",b"src"]) -> None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_direction",b"_direction"]) -> typing.Optional[typing_extensions.Literal["direction"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_edges",b"_edges"]) -> typing.Optional[typing_extensions.Literal["edges"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_depth",b"_max_depth"]) -> typing.Optional[typing_extensions.Literal["max_depth"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_edges",b"_max_edges"]) -> typing.Optional[typing_extensions.Literal["max_edges"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_min_depth",b"_min_depth"]) -> typing.Optional[typing_extensions.Literal["min_depth"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_return_fields",b"_return_fields"]) -> typing.Optional[typing_extensions.Literal["return_fields"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_return_nodes",b"_return_nodes"]) -> typing.Optional[typing_extensions.Literal["return_nodes"]]: ... +global___TraversalRequest = TraversalRequest + +class NodeFilter(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + TYPES_FIELD_NUMBER: builtins.int + MIN_TRAVERSAL_SUCCESSORS_FIELD_NUMBER: builtins.int + MAX_TRAVERSAL_SUCCESSORS_FIELD_NUMBER: builtins.int + types: typing.Text + min_traversal_successors: builtins.int + max_traversal_successors: builtins.int + def __init__(self, + *, + types: typing.Optional[typing.Text] = ..., + min_traversal_successors: typing.Optional[builtins.int] = ..., + max_traversal_successors: typing.Optional[builtins.int] = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_max_traversal_successors",b"_max_traversal_successors","_min_traversal_successors",b"_min_traversal_successors","_types",b"_types","max_traversal_successors",b"max_traversal_successors","min_traversal_successors",b"min_traversal_successors","types",b"types"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_max_traversal_successors",b"_max_traversal_successors","_min_traversal_successors",b"_min_traversal_successors","_types",b"_types","max_traversal_successors",b"max_traversal_successors","min_traversal_successors",b"min_traversal_successors","types",b"types"]) -> None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_max_traversal_successors",b"_max_traversal_successors"]) -> typing.Optional[typing_extensions.Literal["max_traversal_successors"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_min_traversal_successors",b"_min_traversal_successors"]) -> typing.Optional[typing_extensions.Literal["min_traversal_successors"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_types",b"_types"]) -> typing.Optional[typing_extensions.Literal["types"]]: ... +global___NodeFilter = NodeFilter + +class NodeFields(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + SWHID_FIELD_NUMBER: builtins.int + SUCCESSOR_FIELD_NUMBER: builtins.int + SUCCESSOR_SWHID_FIELD_NUMBER: builtins.int + SUCCESSOR_LABEL_FIELD_NUMBER: builtins.int + CNT_LENGTH_FIELD_NUMBER: builtins.int + CNT_IS_SKIPPED_FIELD_NUMBER: builtins.int + REV_AUTHOR_FIELD_NUMBER: builtins.int + REV_AUTHOR_DATE_FIELD_NUMBER: builtins.int + REV_AUTHOR_DATE_OFFSET_FIELD_NUMBER: builtins.int + REV_COMMITTER_FIELD_NUMBER: builtins.int + REV_COMMITTER_DATE_FIELD_NUMBER: builtins.int + REV_COMMITTER_DATE_OFFSET_FIELD_NUMBER: builtins.int + REV_MESSAGE_FIELD_NUMBER: builtins.int + REL_AUTHOR_FIELD_NUMBER: builtins.int + REL_AUTHOR_DATE_FIELD_NUMBER: builtins.int + REL_AUTHOR_DATE_OFFSET_FIELD_NUMBER: builtins.int + REL_NAME_FIELD_NUMBER: builtins.int + REL_MESSAGE_FIELD_NUMBER: builtins.int + ORI_URL_FIELD_NUMBER: builtins.int + swhid: builtins.bool + successor: builtins.bool + successor_swhid: builtins.bool + successor_label: builtins.bool + cnt_length: builtins.bool + cnt_is_skipped: builtins.bool + rev_author: builtins.bool + rev_author_date: builtins.bool + rev_author_date_offset: builtins.bool + rev_committer: builtins.bool + rev_committer_date: builtins.bool + rev_committer_date_offset: builtins.bool + rev_message: builtins.bool + rel_author: builtins.bool + rel_author_date: builtins.bool + rel_author_date_offset: builtins.bool + rel_name: builtins.bool + rel_message: builtins.bool + ori_url: builtins.bool + def __init__(self, + *, + swhid: typing.Optional[builtins.bool] = ..., + successor: typing.Optional[builtins.bool] = ..., + successor_swhid: typing.Optional[builtins.bool] = ..., + successor_label: typing.Optional[builtins.bool] = ..., + cnt_length: typing.Optional[builtins.bool] = ..., + cnt_is_skipped: typing.Optional[builtins.bool] = ..., + rev_author: typing.Optional[builtins.bool] = ..., + rev_author_date: typing.Optional[builtins.bool] = ..., + rev_author_date_offset: typing.Optional[builtins.bool] = ..., + rev_committer: typing.Optional[builtins.bool] = ..., + rev_committer_date: typing.Optional[builtins.bool] = ..., + rev_committer_date_offset: typing.Optional[builtins.bool] = ..., + rev_message: typing.Optional[builtins.bool] = ..., + rel_author: typing.Optional[builtins.bool] = ..., + rel_author_date: typing.Optional[builtins.bool] = ..., + rel_author_date_offset: typing.Optional[builtins.bool] = ..., + rel_name: typing.Optional[builtins.bool] = ..., + rel_message: typing.Optional[builtins.bool] = ..., + ori_url: typing.Optional[builtins.bool] = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_cnt_is_skipped",b"_cnt_is_skipped","_cnt_length",b"_cnt_length","_ori_url",b"_ori_url","_rel_author",b"_rel_author","_rel_author_date",b"_rel_author_date","_rel_author_date_offset",b"_rel_author_date_offset","_rel_message",b"_rel_message","_rel_name",b"_rel_name","_rev_author",b"_rev_author","_rev_author_date",b"_rev_author_date","_rev_author_date_offset",b"_rev_author_date_offset","_rev_committer",b"_rev_committer","_rev_committer_date",b"_rev_committer_date","_rev_committer_date_offset",b"_rev_committer_date_offset","_rev_message",b"_rev_message","_successor",b"_successor","_successor_label",b"_successor_label","_successor_swhid",b"_successor_swhid","_swhid",b"_swhid","cnt_is_skipped",b"cnt_is_skipped","cnt_length",b"cnt_length","ori_url",b"ori_url","rel_author",b"rel_author","rel_author_date",b"rel_author_date","rel_author_date_offset",b"rel_author_date_offset","rel_message",b"rel_message","rel_name",b"rel_name","rev_author",b"rev_author","rev_author_date",b"rev_author_date","rev_author_date_offset",b"rev_author_date_offset","rev_committer",b"rev_committer","rev_committer_date",b"rev_committer_date","rev_committer_date_offset",b"rev_committer_date_offset","rev_message",b"rev_message","successor",b"successor","successor_label",b"successor_label","successor_swhid",b"successor_swhid","swhid",b"swhid"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_cnt_is_skipped",b"_cnt_is_skipped","_cnt_length",b"_cnt_length","_ori_url",b"_ori_url","_rel_author",b"_rel_author","_rel_author_date",b"_rel_author_date","_rel_author_date_offset",b"_rel_author_date_offset","_rel_message",b"_rel_message","_rel_name",b"_rel_name","_rev_author",b"_rev_author","_rev_author_date",b"_rev_author_date","_rev_author_date_offset",b"_rev_author_date_offset","_rev_committer",b"_rev_committer","_rev_committer_date",b"_rev_committer_date","_rev_committer_date_offset",b"_rev_committer_date_offset","_rev_message",b"_rev_message","_successor",b"_successor","_successor_label",b"_successor_label","_successor_swhid",b"_successor_swhid","_swhid",b"_swhid","cnt_is_skipped",b"cnt_is_skipped","cnt_length",b"cnt_length","ori_url",b"ori_url","rel_author",b"rel_author","rel_author_date",b"rel_author_date","rel_author_date_offset",b"rel_author_date_offset","rel_message",b"rel_message","rel_name",b"rel_name","rev_author",b"rev_author","rev_author_date",b"rev_author_date","rev_author_date_offset",b"rev_author_date_offset","rev_committer",b"rev_committer","rev_committer_date",b"rev_committer_date","rev_committer_date_offset",b"rev_committer_date_offset","rev_message",b"rev_message","successor",b"successor","successor_label",b"successor_label","successor_swhid",b"successor_swhid","swhid",b"swhid"]) -> None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_cnt_is_skipped",b"_cnt_is_skipped"]) -> typing.Optional[typing_extensions.Literal["cnt_is_skipped"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_cnt_length",b"_cnt_length"]) -> typing.Optional[typing_extensions.Literal["cnt_length"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_ori_url",b"_ori_url"]) -> typing.Optional[typing_extensions.Literal["ori_url"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_author",b"_rel_author"]) -> typing.Optional[typing_extensions.Literal["rel_author"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_author_date",b"_rel_author_date"]) -> typing.Optional[typing_extensions.Literal["rel_author_date"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_author_date_offset",b"_rel_author_date_offset"]) -> typing.Optional[typing_extensions.Literal["rel_author_date_offset"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_message",b"_rel_message"]) -> typing.Optional[typing_extensions.Literal["rel_message"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_name",b"_rel_name"]) -> typing.Optional[typing_extensions.Literal["rel_name"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_author",b"_rev_author"]) -> typing.Optional[typing_extensions.Literal["rev_author"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_author_date",b"_rev_author_date"]) -> typing.Optional[typing_extensions.Literal["rev_author_date"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_author_date_offset",b"_rev_author_date_offset"]) -> typing.Optional[typing_extensions.Literal["rev_author_date_offset"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_committer",b"_rev_committer"]) -> typing.Optional[typing_extensions.Literal["rev_committer"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_committer_date",b"_rev_committer_date"]) -> typing.Optional[typing_extensions.Literal["rev_committer_date"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_committer_date_offset",b"_rev_committer_date_offset"]) -> typing.Optional[typing_extensions.Literal["rev_committer_date_offset"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_message",b"_rev_message"]) -> typing.Optional[typing_extensions.Literal["rev_message"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_successor",b"_successor"]) -> typing.Optional[typing_extensions.Literal["successor"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_successor_label",b"_successor_label"]) -> typing.Optional[typing_extensions.Literal["successor_label"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_successor_swhid",b"_successor_swhid"]) -> typing.Optional[typing_extensions.Literal["successor_swhid"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_swhid",b"_swhid"]) -> typing.Optional[typing_extensions.Literal["swhid"]]: ... +global___NodeFields = NodeFields + +class Node(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + SWHID_FIELD_NUMBER: builtins.int + SUCCESSOR_FIELD_NUMBER: builtins.int + CNT_LENGTH_FIELD_NUMBER: builtins.int + CNT_IS_SKIPPED_FIELD_NUMBER: builtins.int + REV_AUTHOR_FIELD_NUMBER: builtins.int + REV_AUTHOR_DATE_FIELD_NUMBER: builtins.int + REV_AUTHOR_DATE_OFFSET_FIELD_NUMBER: builtins.int + REV_COMMITTER_FIELD_NUMBER: builtins.int + REV_COMMITTER_DATE_FIELD_NUMBER: builtins.int + REV_COMMITTER_DATE_OFFSET_FIELD_NUMBER: builtins.int + REV_MESSAGE_FIELD_NUMBER: builtins.int + REL_AUTHOR_FIELD_NUMBER: builtins.int + REL_AUTHOR_DATE_FIELD_NUMBER: builtins.int + REL_AUTHOR_DATE_OFFSET_FIELD_NUMBER: builtins.int + REL_NAME_FIELD_NUMBER: builtins.int + REL_MESSAGE_FIELD_NUMBER: builtins.int + ORI_URL_FIELD_NUMBER: builtins.int + swhid: typing.Text + @property + def successor(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Successor]: ... + cnt_length: builtins.int + cnt_is_skipped: builtins.bool + rev_author: builtins.int + rev_author_date: builtins.int + rev_author_date_offset: builtins.int + rev_committer: builtins.int + rev_committer_date: builtins.int + rev_committer_date_offset: builtins.int + rev_message: builtins.bytes + rel_author: builtins.int + rel_author_date: builtins.int + rel_author_date_offset: builtins.int + rel_name: builtins.bytes + rel_message: builtins.bytes + ori_url: typing.Text + def __init__(self, + *, + swhid: typing.Text = ..., + successor: typing.Optional[typing.Iterable[global___Successor]] = ..., + cnt_length: typing.Optional[builtins.int] = ..., + cnt_is_skipped: typing.Optional[builtins.bool] = ..., + rev_author: typing.Optional[builtins.int] = ..., + rev_author_date: typing.Optional[builtins.int] = ..., + rev_author_date_offset: typing.Optional[builtins.int] = ..., + rev_committer: typing.Optional[builtins.int] = ..., + rev_committer_date: typing.Optional[builtins.int] = ..., + rev_committer_date_offset: typing.Optional[builtins.int] = ..., + rev_message: typing.Optional[builtins.bytes] = ..., + rel_author: typing.Optional[builtins.int] = ..., + rel_author_date: typing.Optional[builtins.int] = ..., + rel_author_date_offset: typing.Optional[builtins.int] = ..., + rel_name: typing.Optional[builtins.bytes] = ..., + rel_message: typing.Optional[builtins.bytes] = ..., + ori_url: typing.Optional[typing.Text] = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_cnt_is_skipped",b"_cnt_is_skipped","_cnt_length",b"_cnt_length","_ori_url",b"_ori_url","_rel_author",b"_rel_author","_rel_author_date",b"_rel_author_date","_rel_author_date_offset",b"_rel_author_date_offset","_rel_message",b"_rel_message","_rel_name",b"_rel_name","_rev_author",b"_rev_author","_rev_author_date",b"_rev_author_date","_rev_author_date_offset",b"_rev_author_date_offset","_rev_committer",b"_rev_committer","_rev_committer_date",b"_rev_committer_date","_rev_committer_date_offset",b"_rev_committer_date_offset","_rev_message",b"_rev_message","cnt_is_skipped",b"cnt_is_skipped","cnt_length",b"cnt_length","ori_url",b"ori_url","rel_author",b"rel_author","rel_author_date",b"rel_author_date","rel_author_date_offset",b"rel_author_date_offset","rel_message",b"rel_message","rel_name",b"rel_name","rev_author",b"rev_author","rev_author_date",b"rev_author_date","rev_author_date_offset",b"rev_author_date_offset","rev_committer",b"rev_committer","rev_committer_date",b"rev_committer_date","rev_committer_date_offset",b"rev_committer_date_offset","rev_message",b"rev_message"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_cnt_is_skipped",b"_cnt_is_skipped","_cnt_length",b"_cnt_length","_ori_url",b"_ori_url","_rel_author",b"_rel_author","_rel_author_date",b"_rel_author_date","_rel_author_date_offset",b"_rel_author_date_offset","_rel_message",b"_rel_message","_rel_name",b"_rel_name","_rev_author",b"_rev_author","_rev_author_date",b"_rev_author_date","_rev_author_date_offset",b"_rev_author_date_offset","_rev_committer",b"_rev_committer","_rev_committer_date",b"_rev_committer_date","_rev_committer_date_offset",b"_rev_committer_date_offset","_rev_message",b"_rev_message","cnt_is_skipped",b"cnt_is_skipped","cnt_length",b"cnt_length","ori_url",b"ori_url","rel_author",b"rel_author","rel_author_date",b"rel_author_date","rel_author_date_offset",b"rel_author_date_offset","rel_message",b"rel_message","rel_name",b"rel_name","rev_author",b"rev_author","rev_author_date",b"rev_author_date","rev_author_date_offset",b"rev_author_date_offset","rev_committer",b"rev_committer","rev_committer_date",b"rev_committer_date","rev_committer_date_offset",b"rev_committer_date_offset","rev_message",b"rev_message","successor",b"successor","swhid",b"swhid"]) -> None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_cnt_is_skipped",b"_cnt_is_skipped"]) -> typing.Optional[typing_extensions.Literal["cnt_is_skipped"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_cnt_length",b"_cnt_length"]) -> typing.Optional[typing_extensions.Literal["cnt_length"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_ori_url",b"_ori_url"]) -> typing.Optional[typing_extensions.Literal["ori_url"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_author",b"_rel_author"]) -> typing.Optional[typing_extensions.Literal["rel_author"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_author_date",b"_rel_author_date"]) -> typing.Optional[typing_extensions.Literal["rel_author_date"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_author_date_offset",b"_rel_author_date_offset"]) -> typing.Optional[typing_extensions.Literal["rel_author_date_offset"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_message",b"_rel_message"]) -> typing.Optional[typing_extensions.Literal["rel_message"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rel_name",b"_rel_name"]) -> typing.Optional[typing_extensions.Literal["rel_name"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_author",b"_rev_author"]) -> typing.Optional[typing_extensions.Literal["rev_author"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_author_date",b"_rev_author_date"]) -> typing.Optional[typing_extensions.Literal["rev_author_date"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_author_date_offset",b"_rev_author_date_offset"]) -> typing.Optional[typing_extensions.Literal["rev_author_date_offset"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_committer",b"_rev_committer"]) -> typing.Optional[typing_extensions.Literal["rev_committer"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_committer_date",b"_rev_committer_date"]) -> typing.Optional[typing_extensions.Literal["rev_committer_date"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_committer_date_offset",b"_rev_committer_date_offset"]) -> typing.Optional[typing_extensions.Literal["rev_committer_date_offset"]]: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_rev_message",b"_rev_message"]) -> typing.Optional[typing_extensions.Literal["rev_message"]]: ... +global___Node = Node + +class Successor(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + SWHID_FIELD_NUMBER: builtins.int + LABEL_FIELD_NUMBER: builtins.int + swhid: typing.Text + @property + def label(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___EdgeLabel]: ... + def __init__(self, + *, + swhid: typing.Optional[typing.Text] = ..., + label: typing.Optional[typing.Iterable[global___EdgeLabel]] = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_swhid",b"_swhid","swhid",b"swhid"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_swhid",b"_swhid","label",b"label","swhid",b"swhid"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["_swhid",b"_swhid"]) -> typing.Optional[typing_extensions.Literal["swhid"]]: ... +global___Successor = Successor + +class EdgeLabel(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + NAME_FIELD_NUMBER: builtins.int + PERMISSION_FIELD_NUMBER: builtins.int + name: builtins.bytes + permission: builtins.int + def __init__(self, + *, + name: builtins.bytes = ..., + permission: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["name",b"name","permission",b"permission"]) -> None: ... +global___EdgeLabel = EdgeLabel + +class CountResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + COUNT_FIELD_NUMBER: builtins.int + count: builtins.int + def __init__(self, + *, + count: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["count",b"count"]) -> None: ... +global___CountResponse = CountResponse + +class StatsRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + def __init__(self, + ) -> None: ... +global___StatsRequest = StatsRequest + +class StatsResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + NUM_NODES_FIELD_NUMBER: builtins.int + NUM_EDGES_FIELD_NUMBER: builtins.int + COMPRESSION_FIELD_NUMBER: builtins.int + BITS_PER_NODE_FIELD_NUMBER: builtins.int + BITS_PER_EDGE_FIELD_NUMBER: builtins.int + AVG_LOCALITY_FIELD_NUMBER: builtins.int + INDEGREE_MIN_FIELD_NUMBER: builtins.int + INDEGREE_MAX_FIELD_NUMBER: builtins.int + INDEGREE_AVG_FIELD_NUMBER: builtins.int + OUTDEGREE_MIN_FIELD_NUMBER: builtins.int + OUTDEGREE_MAX_FIELD_NUMBER: builtins.int + OUTDEGREE_AVG_FIELD_NUMBER: builtins.int + num_nodes: builtins.int + num_edges: builtins.int + compression: builtins.float + bits_per_node: builtins.float + bits_per_edge: builtins.float + avg_locality: builtins.float + indegree_min: builtins.int + indegree_max: builtins.int + indegree_avg: builtins.float + outdegree_min: builtins.int + outdegree_max: builtins.int + outdegree_avg: builtins.float + def __init__(self, + *, + num_nodes: builtins.int = ..., + num_edges: builtins.int = ..., + compression: builtins.float = ..., + bits_per_node: builtins.float = ..., + bits_per_edge: builtins.float = ..., + avg_locality: builtins.float = ..., + indegree_min: builtins.int = ..., + indegree_max: builtins.int = ..., + indegree_avg: builtins.float = ..., + outdegree_min: builtins.int = ..., + outdegree_max: builtins.int = ..., + outdegree_avg: builtins.float = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["avg_locality",b"avg_locality","bits_per_edge",b"bits_per_edge","bits_per_node",b"bits_per_node","compression",b"compression","indegree_avg",b"indegree_avg","indegree_max",b"indegree_max","indegree_min",b"indegree_min","num_edges",b"num_edges","num_nodes",b"num_nodes","outdegree_avg",b"outdegree_avg","outdegree_max",b"outdegree_max","outdegree_min",b"outdegree_min"]) -> None: ... +global___StatsResponse = StatsResponse + +class CheckSwhidRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + SWHID_FIELD_NUMBER: builtins.int + swhid: typing.Text + def __init__(self, + *, + swhid: typing.Text = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["swhid",b"swhid"]) -> None: ... +global___CheckSwhidRequest = CheckSwhidRequest + +class CheckSwhidResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + EXISTS_FIELD_NUMBER: builtins.int + DETAILS_FIELD_NUMBER: builtins.int + exists: builtins.bool + details: typing.Text + def __init__(self, + *, + exists: builtins.bool = ..., + details: typing.Text = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["details",b"details","exists",b"exists"]) -> None: ... +global___CheckSwhidResponse = CheckSwhidResponse diff --git a/swh/graph/rpc/swhgraph_pb2_grpc.py b/swh/graph/rpc/swhgraph_pb2_grpc.py new file mode 100644 --- /dev/null +++ b/swh/graph/rpc/swhgraph_pb2_grpc.py @@ -0,0 +1,198 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from swh.graph.rpc import swhgraph_pb2 as swh_dot_graph_dot_rpc_dot_swhgraph__pb2 + + +class TraversalServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.Traverse = channel.unary_stream( + '/swh.graph.TraversalService/Traverse', + request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, + response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.FromString, + ) + self.CountNodes = channel.unary_unary( + '/swh.graph.TraversalService/CountNodes', + request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, + response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.FromString, + ) + self.CountEdges = channel.unary_unary( + '/swh.graph.TraversalService/CountEdges', + request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, + response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.FromString, + ) + self.Stats = channel.unary_unary( + '/swh.graph.TraversalService/Stats', + request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsRequest.SerializeToString, + response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsResponse.FromString, + ) + self.CheckSwhid = channel.unary_unary( + '/swh.graph.TraversalService/CheckSwhid', + request_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CheckSwhidRequest.SerializeToString, + response_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CheckSwhidResponse.FromString, + ) + + +class TraversalServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def Traverse(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def CountNodes(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def CountEdges(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Stats(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def CheckSwhid(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_TraversalServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'Traverse': grpc.unary_stream_rpc_method_handler( + servicer.Traverse, + request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.FromString, + response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.SerializeToString, + ), + 'CountNodes': grpc.unary_unary_rpc_method_handler( + servicer.CountNodes, + request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.FromString, + response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.SerializeToString, + ), + 'CountEdges': grpc.unary_unary_rpc_method_handler( + servicer.CountEdges, + request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.FromString, + response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.SerializeToString, + ), + 'Stats': grpc.unary_unary_rpc_method_handler( + servicer.Stats, + request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsRequest.FromString, + response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsResponse.SerializeToString, + ), + 'CheckSwhid': grpc.unary_unary_rpc_method_handler( + servicer.CheckSwhid, + request_deserializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CheckSwhidRequest.FromString, + response_serializer=swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CheckSwhidResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'swh.graph.TraversalService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class TraversalService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def Traverse(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/swh.graph.TraversalService/Traverse', + swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, + swh_dot_graph_dot_rpc_dot_swhgraph__pb2.Node.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def CountNodes(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/CountNodes', + swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, + swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def CountEdges(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/CountEdges', + swh_dot_graph_dot_rpc_dot_swhgraph__pb2.TraversalRequest.SerializeToString, + swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CountResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Stats(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/Stats', + swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsRequest.SerializeToString, + swh_dot_graph_dot_rpc_dot_swhgraph__pb2.StatsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def CheckSwhid(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/swh.graph.TraversalService/CheckSwhid', + swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CheckSwhidRequest.SerializeToString, + swh_dot_graph_dot_rpc_dot_swhgraph__pb2.CheckSwhidResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/swh/graph/rpc_server.py b/swh/graph/rpc_server.py new file mode 100644 --- /dev/null +++ b/swh/graph/rpc_server.py @@ -0,0 +1,33 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +""" +A simple tool to start the swh-graph GRPC server in Java. +""" + +import subprocess + +import aiohttp.test_utils +import aiohttp.web + +from swh.graph.config import check_config + + +def spawn_java_rpc_server(config, port=None): + if port is None: + port = aiohttp.test_utils.unused_port() + config = check_config(config or {}) + cmd = [ + "java", + "-cp", + config["classpath"], + *config["java_tool_options"].split(), + "org.softwareheritage.graph.rpc.GraphServer", + "--port", + str(port), + config["graph"]["path"], + ] + server = subprocess.Popen(cmd) + return server, port diff --git a/swh/graph/server/__init__.py b/swh/graph/server/__init__.py deleted file mode 100644 diff --git a/swh/graph/tests/conftest.py b/swh/graph/tests/conftest.py --- a/swh/graph/tests/conftest.py +++ b/swh/graph/tests/conftest.py @@ -10,8 +10,8 @@ from aiohttp.test_utils import TestClient, TestServer, loop_context import pytest -from swh.graph.client import RemoteGraphClient -from swh.graph.naive_client import NaiveClient +from swh.graph.http_client import RemoteGraphClient +from swh.graph.http_naive_client import NaiveClient SWH_GRAPH_TESTS_ROOT = Path(__file__).parents[0] TEST_GRAPH_PATH = SWH_GRAPH_TESTS_ROOT / "dataset/compressed/example" @@ -24,13 +24,12 @@ def run(self): # Lazy import to allow debian packaging - from swh.graph.backend import Backend - from swh.graph.server.app import make_app + from swh.graph.http_server import make_app try: - backend = Backend(graph_path=str(TEST_GRAPH_PATH)) + config = {"graph": {"path": TEST_GRAPH_PATH}} with loop_context() as loop: - app = make_app(backend=backend, debug=True) + app = make_app(config=config, debug=True) client = TestClient(TestServer(app), loop=loop) loop.run_until_complete(client.start_server()) url = client.make_url("/graph/") diff --git a/swh/graph/tests/test_api_client.py b/swh/graph/tests/test_http_client.py rename from swh/graph/tests/test_api_client.py rename to swh/graph/tests/test_http_client.py --- a/swh/graph/tests/test_api_client.py +++ b/swh/graph/tests/test_http_client.py @@ -4,7 +4,7 @@ from pytest import raises from swh.core.api import RemoteException -from swh.graph.client import GraphArgumentException +from swh.graph.http_client import GraphArgumentException TEST_ORIGIN_ID = "swh:1:ori:{}".format( hashlib.sha1(b"https://example.com/swh/graph").hexdigest() @@ -13,31 +13,18 @@ def test_stats(graph_client): stats = graph_client.stats() - - assert set(stats.keys()) == {"counts", "ratios", "indegree", "outdegree"} - - assert set(stats["counts"].keys()) == {"nodes", "edges"} - assert set(stats["ratios"].keys()) == { - "compression", - "bits_per_node", - "bits_per_edge", - "avg_locality", - } - assert set(stats["indegree"].keys()) == {"min", "max", "avg"} - assert set(stats["outdegree"].keys()) == {"min", "max", "avg"} - - assert stats["counts"]["nodes"] == 21 - assert stats["counts"]["edges"] == 23 - assert isinstance(stats["ratios"]["compression"], float) - assert isinstance(stats["ratios"]["bits_per_node"], float) - assert isinstance(stats["ratios"]["bits_per_edge"], float) - assert isinstance(stats["ratios"]["avg_locality"], float) - assert stats["indegree"]["min"] == 0 - assert stats["indegree"]["max"] == 3 - assert isinstance(stats["indegree"]["avg"], float) - assert stats["outdegree"]["min"] == 0 - assert stats["outdegree"]["max"] == 3 - assert isinstance(stats["outdegree"]["avg"], float) + assert stats["num_nodes"] == 21 + assert stats["num_edges"] == 23 + assert isinstance(stats["compression"], float) + assert isinstance(stats["bits_per_node"], float) + assert isinstance(stats["bits_per_edge"], float) + assert isinstance(stats["avg_locality"], float) + assert stats["indegree_min"] == 0 + assert stats["indegree_max"] == 3 + assert isinstance(stats["indegree_avg"], float) + assert stats["outdegree_min"] == 0 + assert stats["outdegree_max"] == 3 + assert isinstance(stats["outdegree_avg"], float) def test_leaves(graph_client): @@ -259,6 +246,7 @@ assert set(actual) == set(expected) +@pytest.mark.skip(reason="Random walk is deprecated") def test_random_walk_dst_is_type(graph_client): """as the walk is random, we test a visit from a cnt node to a release reachable from every single path in the backward graph, and only check the @@ -288,6 +276,7 @@ assert len(actual) == 3 +@pytest.mark.skip(reason="Random walk is deprecated") def test_random_walk_dst_is_node(graph_client): """Same as test_random_walk_dst_is_type, but we target the specific release node instead of a type