diff --git a/docs/docker.rst b/docs/docker.rst --- a/docs/docker.rst +++ b/docs/docker.rst @@ -73,3 +73,6 @@ $ java -cp app/swh-graph.jar \ org.softwareheritage.graph.App data/compressed/g + +To specify the port on which the server will run, use the `--port` or `-p` flag +(default is 5009). diff --git a/java/server/README.md b/java/server/README.md --- a/java/server/README.md +++ b/java/server/README.md @@ -14,7 +14,7 @@ -------------- ```bash -$ java -cp target/swh-graph-1.0-jar-with-dependencies.jar \ +$ java -cp target/swh-graph-0.0.1-jar-with-dependencies.jar \ org.softwareheritage.graph.App \ ``` @@ -41,7 +41,7 @@ $ mvn compile assembly:single # Dump mapping files -$ java -cp target/swh-graph-1.0-jar-with-dependencies.jar \ +$ java -cp target/swh-graph-0.0.1-jar-with-dependencies.jar \ org.softwareheritage.graph.backend.Setup \ src/test/dataset/example.nodes.csv.gz \ src/test/dataset/output/example diff --git a/java/server/pom.xml b/java/server/pom.xml --- a/java/server/pom.xml +++ b/java/server/pom.xml @@ -6,7 +6,7 @@ org.softwareheritage.graph swh-graph - 1.0 + 0.0.1 swh-graph https://www.softwareheritage.org/ @@ -59,6 +59,11 @@ fastutil 8.2.2 + + com.martiansoftware + jsap + 2.1 + diff --git a/java/server/src/main/java/org/softwareheritage/graph/AllowedEdges.java b/java/server/src/main/java/org/softwareheritage/graph/AllowedEdges.java --- a/java/server/src/main/java/org/softwareheritage/graph/AllowedEdges.java +++ b/java/server/src/main/java/org/softwareheritage/graph/AllowedEdges.java @@ -9,8 +9,8 @@ * Edge restriction based on node types, used when visiting the graph. * * @author Thibault Allançon - * @version 1.0 - * @since 1.0 + * @version 0.0.1 + * @since 0.0.1 */ public class AllowedEdges { diff --git a/java/server/src/main/java/org/softwareheritage/graph/App.java b/java/server/src/main/java/org/softwareheritage/graph/App.java --- a/java/server/src/main/java/org/softwareheritage/graph/App.java +++ b/java/server/src/main/java/org/softwareheritage/graph/App.java @@ -6,6 +6,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.martiansoftware.jsap.FlaggedOption; +import com.martiansoftware.jsap.JSAP; +import com.martiansoftware.jsap.JSAPException; +import com.martiansoftware.jsap.JSAPResult; +import com.martiansoftware.jsap.Parameter; +import com.martiansoftware.jsap.SimpleJSAP; +import com.martiansoftware.jsap.UnflaggedOption; import io.javalin.Javalin; import io.javalin.http.Context; import io.javalin.plugin.json.JavalinJackson; @@ -19,15 +26,37 @@ * Entrypoint of the swh-graph server REST API. * * @author Thibault Allançon - * @version 1.0 - * @since 1.0 + * @version 0.0.1 + * @since 0.0.1 */ public class App { - public static void main(String[] args) throws IOException { - String path = args[0]; - Graph graph = new Graph(path); - Stats stats = new Stats(path); + public static void main(String[] args) throws IOException, JSAPException { + SimpleJSAP jsap = new SimpleJSAP( + App.class.getName(), + "Server to load and query a compressed graph representation of Software Heritage archive.", + new Parameter[] { + new FlaggedOption("port", JSAP.INTEGER_PARSER, "5009", JSAP.NOT_REQUIRED, 'p', "port", + "Binding port of the server."), + new UnflaggedOption("graphPath", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, + JSAP.NOT_GREEDY, "The basename of the compressed graph."), + } + ); + + JSAPResult config = jsap.parse(args); + if (jsap.messagePrinted()) { + System.exit(1); + } + + String graphPath = config.getString("graphPath"); + int port = config.getInt("port"); + + startServer(graphPath, port); + } + + private static void startServer(String graphPath, int port) throws IOException { + Graph graph = new Graph(graphPath); + Stats stats = new Stats(graphPath); // Clean up on exit Runtime.getRuntime().addShutdownHook(new Thread() { @@ -45,7 +74,7 @@ objectMapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); JavalinJackson.configure(objectMapper); - Javalin app = Javalin.create().start(5009); + Javalin app = Javalin.create().start(port); app.before("/stats/*", ctx -> { checkQueryStrings(ctx, ""); }); app.before("/leaves/*", ctx -> { checkQueryStrings(ctx, "direction|edges"); }); diff --git a/java/server/src/main/java/org/softwareheritage/graph/Endpoint.java b/java/server/src/main/java/org/softwareheritage/graph/Endpoint.java --- a/java/server/src/main/java/org/softwareheritage/graph/Endpoint.java +++ b/java/server/src/main/java/org/softwareheritage/graph/Endpoint.java @@ -14,8 +14,8 @@ * REST API endpoints wrapper functions. * * @author Thibault Allançon - * @version 1.0 - * @since 1.0 + * @version 0.0.1 + * @since 0.0.1 */ public class Endpoint { diff --git a/java/server/src/main/java/org/softwareheritage/graph/Graph.java b/java/server/src/main/java/org/softwareheritage/graph/Graph.java --- a/java/server/src/main/java/org/softwareheritage/graph/Graph.java +++ b/java/server/src/main/java/org/softwareheritage/graph/Graph.java @@ -13,8 +13,8 @@ * Main class storing the compressed graph and node id mappings. * * @author Thibault Allançon - * @version 1.0 - * @since 1.0 + * @version 0.0.1 + * @since 0.0.1 */ public class Graph { diff --git a/java/server/src/main/java/org/softwareheritage/graph/Neighbors.java b/java/server/src/main/java/org/softwareheritage/graph/Neighbors.java --- a/java/server/src/main/java/org/softwareheritage/graph/Neighbors.java +++ b/java/server/src/main/java/org/softwareheritage/graph/Neighbors.java @@ -11,8 +11,8 @@ * Iterator class to go over a node neighbors in the graph. * * @author Thibault Allançon - * @version 1.0 - * @since 1.0 + * @version 0.0.1 + * @since 0.0.1 */ public class Neighbors implements Iterable { @@ -49,8 +49,8 @@ * Inner class for {@link Neighbors} iterator. * * @author Thibault Allançon - * @version 1.0 - * @since 1.0 + * @version 0.0.1 + * @since 0.0.1 */ public class NeighborsIterator implements Iterator { diff --git a/java/server/src/main/java/org/softwareheritage/graph/Node.java b/java/server/src/main/java/org/softwareheritage/graph/Node.java --- a/java/server/src/main/java/org/softwareheritage/graph/Node.java +++ b/java/server/src/main/java/org/softwareheritage/graph/Node.java @@ -8,8 +8,8 @@ * A node in the Software Heritage graph. * * @author Thibault Allançon - * @version 1.0 - * @since 1.0 + * @version 0.0.1 + * @since 0.0.1 */ public class Node { diff --git a/java/server/src/main/java/org/softwareheritage/graph/SwhId.java b/java/server/src/main/java/org/softwareheritage/graph/SwhId.java --- a/java/server/src/main/java/org/softwareheritage/graph/SwhId.java +++ b/java/server/src/main/java/org/softwareheritage/graph/SwhId.java @@ -10,8 +10,8 @@ * identifier documentation. * * @author Thibault Allançon - * @version 1.0 - * @since 1.0 + * @version 0.0.1 + * @since 0.0.1 */ public class SwhId { diff --git a/java/server/src/main/java/org/softwareheritage/graph/SwhPath.java b/java/server/src/main/java/org/softwareheritage/graph/SwhPath.java --- a/java/server/src/main/java/org/softwareheritage/graph/SwhPath.java +++ b/java/server/src/main/java/org/softwareheritage/graph/SwhPath.java @@ -10,8 +10,8 @@ * Wrapper class to store a list of {@link SwhId}. * * @author Thibault Allançon - * @version 1.0 - * @since 1.0 + * @version 0.0.1 + * @since 0.0.1 * @see org.softwareheritage.graph.SwhId */ diff --git a/java/server/src/main/java/org/softwareheritage/graph/algo/Stats.java b/java/server/src/main/java/org/softwareheritage/graph/algo/Stats.java --- a/java/server/src/main/java/org/softwareheritage/graph/algo/Stats.java +++ b/java/server/src/main/java/org/softwareheritage/graph/algo/Stats.java @@ -8,8 +8,8 @@ * Statistics on the compressed graph. * * @author Thibault Allançon - * @version 1.0 - * @since 1.0 + * @version 0.0.1 + * @since 0.0.1 */ public class Stats { diff --git a/java/server/src/main/java/org/softwareheritage/graph/algo/Traversal.java b/java/server/src/main/java/org/softwareheritage/graph/algo/Traversal.java --- a/java/server/src/main/java/org/softwareheritage/graph/algo/Traversal.java +++ b/java/server/src/main/java/org/softwareheritage/graph/algo/Traversal.java @@ -18,8 +18,8 @@ * Traversal algorithms on the compressed graph. * * @author Thibault Allançon - * @version 1.0 - * @since 1.0 + * @version 0.0.1 + * @since 0.0.1 */ public class Traversal { diff --git a/java/server/src/main/java/org/softwareheritage/graph/backend/MapFile.java b/java/server/src/main/java/org/softwareheritage/graph/backend/MapFile.java --- a/java/server/src/main/java/org/softwareheritage/graph/backend/MapFile.java +++ b/java/server/src/main/java/org/softwareheritage/graph/backend/MapFile.java @@ -11,8 +11,8 @@ * Wrapper class around very big mmap()-ed file. * * @author Thibault Allançon - * @version 1.0 - * @since 1.0 + * @version 0.0.1 + * @since 0.0.1 */ public class MapFile { diff --git a/java/server/src/main/java/org/softwareheritage/graph/backend/NodeIdMap.java b/java/server/src/main/java/org/softwareheritage/graph/backend/NodeIdMap.java --- a/java/server/src/main/java/org/softwareheritage/graph/backend/NodeIdMap.java +++ b/java/server/src/main/java/org/softwareheritage/graph/backend/NodeIdMap.java @@ -10,8 +10,8 @@ * Mapping between internal long node id and external SWH PID. * * @author Thibault Allançon - * @version 1.0 - * @since 1.0 + * @version 0.0.1 + * @since 0.0.1 */ public class NodeIdMap { diff --git a/java/server/src/main/java/org/softwareheritage/graph/backend/NodeTypesMap.java b/java/server/src/main/java/org/softwareheritage/graph/backend/NodeTypesMap.java --- a/java/server/src/main/java/org/softwareheritage/graph/backend/NodeTypesMap.java +++ b/java/server/src/main/java/org/softwareheritage/graph/backend/NodeTypesMap.java @@ -13,8 +13,8 @@ * href="https://docs.softwareheritage.org/devel/swh-model/data-model.html">data model. * * @author Thibault Allançon - * @version 1.0 - * @since 1.0 + * @version 0.0.1 + * @since 0.0.1 */ public class NodeTypesMap { diff --git a/java/server/src/main/java/org/softwareheritage/graph/backend/Setup.java b/java/server/src/main/java/org/softwareheritage/graph/backend/Setup.java --- a/java/server/src/main/java/org/softwareheritage/graph/backend/Setup.java +++ b/java/server/src/main/java/org/softwareheritage/graph/backend/Setup.java @@ -28,8 +28,8 @@ * Pre-processing steps (such as dumping mapping files on disk) before running the graph service. * * @author Thibault Allançon - * @version 1.0 - * @since 1.0 + * @version 0.0.1 + * @since 0.0.1 */ public class Setup { diff --git a/java/server/src/main/java/org/softwareheritage/graph/benchmark/LinuxLog.java b/java/server/src/main/java/org/softwareheritage/graph/benchmark/LinuxLog.java --- a/java/server/src/main/java/org/softwareheritage/graph/benchmark/LinuxLog.java +++ b/java/server/src/main/java/org/softwareheritage/graph/benchmark/LinuxLog.java @@ -10,8 +10,8 @@ * Linux git log experiment to benchmark graph traversal. * * @author Thibault Allançon - * @version 1.0 - * @since 1.0 + * @version 0.0.1 + * @since 0.0.1 */ public class LinuxLog { diff --git a/swh/graph/tests/__init__.py b/swh/graph/tests/__init__.py --- a/swh/graph/tests/__init__.py +++ b/swh/graph/tests/__init__.py @@ -0,0 +1 @@ +SWH_GRAPH_VERSION = '0.0.1' diff --git a/swh/graph/tests/test_api_client.py b/swh/graph/tests/test_api_client.py new file mode 100644 --- /dev/null +++ b/swh/graph/tests/test_api_client.py @@ -0,0 +1,151 @@ +from pathlib import Path +import subprocess +import time + +import aiohttp.test_utils +import pytest + +from swh.graph.client import RemoteGraphClient +from swh.graph.tests import SWH_GRAPH_VERSION + + +@pytest.fixture(scope='module') +def graph_client(): + swh_graph_root = Path(__file__).parents[3] + java_dir = swh_graph_root / 'java/server' + + # Compile Java server using maven + pom_path = java_dir / 'pom.xml' + subprocess.run( + ['mvn', '-f', str(pom_path), 'compile', 'assembly:single'], check=True) + + port = aiohttp.test_utils.unused_port() + + # Start Java server + jar_file = 'swh-graph-{}-jar-with-dependencies.jar'.format( + SWH_GRAPH_VERSION) + jar_path = java_dir / 'target' / jar_file + graph_path = java_dir / 'src/test/dataset/output/example' + server = subprocess.Popen([ + 'java', '-cp', str(jar_path), + 'org.softwareheritage.graph.App', str(graph_path), '-p', str(port) + ]) + + # Make sure the server is entirely started before running the client + time.sleep(1) + + # Start Python client + localhost = 'http://0.0.0.0:{}'.format(port) + client = RemoteGraphClient(localhost) + + yield client + + print('Service teardown') + server.kill() + + +class TestEndpoints: + @pytest.fixture(autouse=True) + def init_graph_client(self, graph_client): + self.client = graph_client + + def test_leaves(self): + actual = self.client.leaves( + 'swh:1:ori:0000000000000000000000000000000000000021' + ) + expected = [ + 'swh:1:cnt:0000000000000000000000000000000000000001', + 'swh:1:cnt:0000000000000000000000000000000000000004', + 'swh:1:cnt:0000000000000000000000000000000000000005', + 'swh:1:cnt:0000000000000000000000000000000000000007' + ] + assert set(actual) == set(expected) + + def test_neighbors(self): + actual = self.client.neighbors( + 'swh:1:rev:0000000000000000000000000000000000000009', + direction='backward' + ) + expected = [ + 'swh:1:snp:0000000000000000000000000000000000000020', + 'swh:1:rel:0000000000000000000000000000000000000010', + 'swh:1:rev:0000000000000000000000000000000000000013' + ] + assert set(actual) == set(expected) + + def test_stats(self): + stats = self.client.stats() + + assert set(stats.keys()) == {'counts', 'ratios', 'indegree', + 'outdegree'} + + assert set(stats['counts'].keys()) == {'nodes', 'edges'} + assert set(stats['ratios'].keys()) == {'compression', 'bits_per_node', + 'bits_per_edge', 'avg_locality'} + assert set(stats['indegree'].keys()) == {'min', 'max', 'avg'} + assert set(stats['outdegree'].keys()) == {'min', 'max', 'avg'} + + assert stats['counts']['nodes'] == 21 + assert stats['counts']['edges'] == 23 + assert isinstance(stats['ratios']['compression'], float) + assert isinstance(stats['ratios']['bits_per_node'], float) + assert isinstance(stats['ratios']['bits_per_edge'], float) + assert isinstance(stats['ratios']['avg_locality'], float) + assert stats['indegree']['min'] == 0 + assert stats['indegree']['max'] == 3 + assert isinstance(stats['indegree']['avg'], float) + assert stats['outdegree']['min'] == 0 + assert stats['outdegree']['max'] == 3 + assert isinstance(stats['outdegree']['avg'], float) + + def test_visit_nodes(self): + actual = self.client.visit_nodes( + 'swh:1:rel:0000000000000000000000000000000000000010', + edges='rel:rev,rev:rev' + ) + expected = [ + 'swh:1:rel:0000000000000000000000000000000000000010', + 'swh:1:rev:0000000000000000000000000000000000000009', + 'swh:1:rev:0000000000000000000000000000000000000003' + ] + assert set(actual) == set(expected) + + def test_visit_paths(self): + actual = [tuple(path) for path in + self.client.visit_paths( + 'swh:1:snp:0000000000000000000000000000000000000020', + edges='snp:*,rev:*') + ] + expected = [ + ( + 'swh:1:snp:0000000000000000000000000000000000000020', + 'swh:1:rev:0000000000000000000000000000000000000009', + 'swh:1:rev:0000000000000000000000000000000000000003', + 'swh:1:dir:0000000000000000000000000000000000000002' + ), + ( + 'swh:1:snp:0000000000000000000000000000000000000020', + 'swh:1:rev:0000000000000000000000000000000000000009', + 'swh:1:dir:0000000000000000000000000000000000000008' + ), + ( + 'swh:1:snp:0000000000000000000000000000000000000020', + 'swh:1:rel:0000000000000000000000000000000000000010' + ) + ] + assert set(actual) == set(expected) + + def test_walk(self): + actual = self.client.walk( + 'swh:1:dir:0000000000000000000000000000000000000016', 'rel', + edges='dir:dir,dir:rev,rev:*', + direction='backward', + traversal='bfs' + ) + expected = [ + 'swh:1:dir:0000000000000000000000000000000000000016', + 'swh:1:dir:0000000000000000000000000000000000000017', + 'swh:1:rev:0000000000000000000000000000000000000018', + 'swh:1:rel:0000000000000000000000000000000000000019' + ] + assert set(actual) == set(expected)