diff --git a/MANIFEST.in b/MANIFEST.in index 8c63402..74d2bd1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,7 @@ include Makefile include requirements.txt include requirements-swh.txt include version.txt include README.md recursive-include swh py.typed +recursive-include swh/graph/tests/dataset * diff --git a/java/server/README.md b/java/server/README.md index 9e2e315..f9662a7 100644 --- a/java/server/README.md +++ b/java/server/README.md @@ -1,50 +1,50 @@ Graph service - Server side =========================== Server side Java REST API. Build ----- ```bash $ mvn compile assembly:single ``` Start REST API -------------- ```bash $ java -cp target/swh-graph-jar-with-dependencies.jar \ org.softwareheritage.graph.App \ ``` Default port is 5009 (use the `--port` option to change port number). If you need timings metadata send back to the client in addition to the result, use the `--timings` flag. Tests ----- Unit tests rely on test data that are already available in the Git repository -(under `src/test/dataset/`). You generally only need to run them using Maven: +(under `src/swh/graph/tests/dataset/`). You generally only need to run them using Maven: ```bash $ mvn test ``` In case you want to regenerate the test data: ```bash # Graph compression -$ cd src/test/dataset +$ cd src/swh/graph/tests/dataset $ ./generate_graph.sh -$ cd ../../../ +$ cd ../../../.. $ mvn compile assembly:single # Dump mapping files $ java -cp target/swh-graph-jar-with-dependencies.jar \ org.softwareheritage.graph.backend.Setup \ - src/test/dataset/example.nodes.csv.gz \ - src/test/dataset/output/example + src/swh/graph/tests/dataset/example.nodes.csv.gz \ + src/swh/graph/tests/dataset/output/example ``` diff --git a/java/server/src/test/java/org/softwareheritage/graph/GraphTest.java b/java/server/src/test/java/org/softwareheritage/graph/GraphTest.java index 9355989..7962f61 100644 --- a/java/server/src/test/java/org/softwareheritage/graph/GraphTest.java +++ b/java/server/src/test/java/org/softwareheritage/graph/GraphTest.java @@ -1,30 +1,30 @@ package org.softwareheritage.graph; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collection; import org.junit.Assert; import org.junit.BeforeClass; import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; import org.softwareheritage.graph.Graph; public class GraphTest { static Graph graph; public static void assertEqualsAnyOrder(Collection expecteds, Collection actuals) { Assert.assertThat(expecteds, containsInAnyOrder(actuals.toArray())); } @BeforeClass public static void setUp() throws IOException { - Path graphPath = Paths.get("..", "..", "tests", "dataset", "output", "example"); + Path graphPath = Paths.get("..", "..", "swh", "graph", "tests", "dataset", "output", "example"); graph = new Graph(graphPath.toString()); } public Graph getGraph() { return graph; } } diff --git a/swh/graph/tests/conftest.py b/swh/graph/tests/conftest.py index 1c391a8..76fbf86 100644 --- a/swh/graph/tests/conftest.py +++ b/swh/graph/tests/conftest.py @@ -1,46 +1,46 @@ import multiprocessing import pytest -from pathlib import Path from aiohttp.test_utils import TestServer, TestClient, loop_context +from pathlib import Path from swh.graph.graph import load as graph_load from swh.graph.client import RemoteGraphClient from swh.graph.backend import Backend from swh.graph.server.app import make_app -SWH_GRAPH_ROOT = Path(__file__).parents[3] -TEST_GRAPH_PATH = SWH_GRAPH_ROOT / 'tests/dataset/output/example' +SWH_GRAPH_TESTS_ROOT = Path(__file__).parents[0] +TEST_GRAPH_PATH = SWH_GRAPH_TESTS_ROOT / 'dataset/output/example' class GraphServerProcess(multiprocessing.Process): def __init__(self, q, *args, **kwargs): self.q = q super().__init__(*args, **kwargs) def run(self): backend = Backend(graph_path=str(TEST_GRAPH_PATH)) with backend: with loop_context() as loop: app = make_app(backend=backend, debug=True) client = TestClient(TestServer(app), loop=loop) loop.run_until_complete(client.start_server()) url = client.make_url('/graph/') self.q.put(url) loop.run_forever() @pytest.fixture(scope="module") def graph_client(): queue = multiprocessing.Queue() server = GraphServerProcess(queue) server.start() url = queue.get() yield RemoteGraphClient(str(url)) server.terminate() @pytest.fixture(scope="module") def graph(): with graph_load(str(TEST_GRAPH_PATH)) as g: yield g diff --git a/tests/dataset/.gitignore b/swh/graph/tests/dataset/.gitignore similarity index 100% rename from tests/dataset/.gitignore rename to swh/graph/tests/dataset/.gitignore diff --git a/tests/dataset/example.edges.csv b/swh/graph/tests/dataset/example.edges.csv similarity index 100% rename from tests/dataset/example.edges.csv rename to swh/graph/tests/dataset/example.edges.csv diff --git a/tests/dataset/example.edges.csv.gz b/swh/graph/tests/dataset/example.edges.csv.gz similarity index 100% rename from tests/dataset/example.edges.csv.gz rename to swh/graph/tests/dataset/example.edges.csv.gz diff --git a/tests/dataset/example.nodes.csv b/swh/graph/tests/dataset/example.nodes.csv similarity index 100% rename from tests/dataset/example.nodes.csv rename to swh/graph/tests/dataset/example.nodes.csv diff --git a/tests/dataset/example.nodes.csv.gz b/swh/graph/tests/dataset/example.nodes.csv.gz similarity index 100% rename from tests/dataset/example.nodes.csv.gz rename to swh/graph/tests/dataset/example.nodes.csv.gz diff --git a/tests/dataset/generate_graph.sh b/swh/graph/tests/dataset/generate_graph.sh similarity index 100% rename from tests/dataset/generate_graph.sh rename to swh/graph/tests/dataset/generate_graph.sh diff --git a/tests/dataset/img/.gitignore b/swh/graph/tests/dataset/img/.gitignore similarity index 100% rename from tests/dataset/img/.gitignore rename to swh/graph/tests/dataset/img/.gitignore diff --git a/tests/dataset/img/Makefile b/swh/graph/tests/dataset/img/Makefile similarity index 100% rename from tests/dataset/img/Makefile rename to swh/graph/tests/dataset/img/Makefile diff --git a/tests/dataset/img/example.dot b/swh/graph/tests/dataset/img/example.dot similarity index 100% rename from tests/dataset/img/example.dot rename to swh/graph/tests/dataset/img/example.dot diff --git a/tests/dataset/output/example-transposed.graph b/swh/graph/tests/dataset/output/example-transposed.graph similarity index 100% rename from tests/dataset/output/example-transposed.graph rename to swh/graph/tests/dataset/output/example-transposed.graph diff --git a/tests/dataset/output/example-transposed.obl b/swh/graph/tests/dataset/output/example-transposed.obl similarity index 100% rename from tests/dataset/output/example-transposed.obl rename to swh/graph/tests/dataset/output/example-transposed.obl diff --git a/tests/dataset/output/example-transposed.offsets b/swh/graph/tests/dataset/output/example-transposed.offsets similarity index 100% rename from tests/dataset/output/example-transposed.offsets rename to swh/graph/tests/dataset/output/example-transposed.offsets diff --git a/tests/dataset/output/example-transposed.properties b/swh/graph/tests/dataset/output/example-transposed.properties similarity index 100% rename from tests/dataset/output/example-transposed.properties rename to swh/graph/tests/dataset/output/example-transposed.properties diff --git a/tests/dataset/output/example.graph b/swh/graph/tests/dataset/output/example.graph similarity index 100% rename from tests/dataset/output/example.graph rename to swh/graph/tests/dataset/output/example.graph diff --git a/tests/dataset/output/example.indegree b/swh/graph/tests/dataset/output/example.indegree similarity index 100% rename from tests/dataset/output/example.indegree rename to swh/graph/tests/dataset/output/example.indegree diff --git a/tests/dataset/output/example.mph b/swh/graph/tests/dataset/output/example.mph similarity index 100% rename from tests/dataset/output/example.mph rename to swh/graph/tests/dataset/output/example.mph diff --git a/tests/dataset/output/example.node2pid.bin b/swh/graph/tests/dataset/output/example.node2pid.bin similarity index 100% rename from tests/dataset/output/example.node2pid.bin rename to swh/graph/tests/dataset/output/example.node2pid.bin diff --git a/tests/dataset/output/example.node2pid.csv b/swh/graph/tests/dataset/output/example.node2pid.csv similarity index 100% rename from tests/dataset/output/example.node2pid.csv rename to swh/graph/tests/dataset/output/example.node2pid.csv diff --git a/tests/dataset/output/example.node2type.map b/swh/graph/tests/dataset/output/example.node2type.map similarity index 100% rename from tests/dataset/output/example.node2type.map rename to swh/graph/tests/dataset/output/example.node2type.map diff --git a/tests/dataset/output/example.obl b/swh/graph/tests/dataset/output/example.obl similarity index 100% rename from tests/dataset/output/example.obl rename to swh/graph/tests/dataset/output/example.obl diff --git a/tests/dataset/output/example.offsets b/swh/graph/tests/dataset/output/example.offsets similarity index 100% rename from tests/dataset/output/example.offsets rename to swh/graph/tests/dataset/output/example.offsets diff --git a/tests/dataset/output/example.order b/swh/graph/tests/dataset/output/example.order similarity index 100% rename from tests/dataset/output/example.order rename to swh/graph/tests/dataset/output/example.order diff --git a/tests/dataset/output/example.outdegree b/swh/graph/tests/dataset/output/example.outdegree similarity index 100% rename from tests/dataset/output/example.outdegree rename to swh/graph/tests/dataset/output/example.outdegree diff --git a/tests/dataset/output/example.pid2node.bin b/swh/graph/tests/dataset/output/example.pid2node.bin similarity index 100% rename from tests/dataset/output/example.pid2node.bin rename to swh/graph/tests/dataset/output/example.pid2node.bin diff --git a/tests/dataset/output/example.pid2node.csv b/swh/graph/tests/dataset/output/example.pid2node.csv similarity index 100% rename from tests/dataset/output/example.pid2node.csv rename to swh/graph/tests/dataset/output/example.pid2node.csv diff --git a/tests/dataset/output/example.properties b/swh/graph/tests/dataset/output/example.properties similarity index 100% rename from tests/dataset/output/example.properties rename to swh/graph/tests/dataset/output/example.properties diff --git a/tests/dataset/output/example.stats b/swh/graph/tests/dataset/output/example.stats similarity index 100% rename from tests/dataset/output/example.stats rename to swh/graph/tests/dataset/output/example.stats diff --git a/swh/graph/tests/test_api_client.py b/swh/graph/tests/test_api_client.py index 40c32d2..89ece0f 100644 --- a/swh/graph/tests/test_api_client.py +++ b/swh/graph/tests/test_api_client.py @@ -1,129 +1,121 @@ -import os -import pytest - - -pytestmark = pytest.mark.skipif(os.environ.get('JENKINS'), reason='T2055') - - def test_stats(graph_client): stats = graph_client.stats() assert set(stats.keys()) == {'counts', 'ratios', 'indegree', 'outdegree'} assert set(stats['counts'].keys()) == {'nodes', 'edges'} assert set(stats['ratios'].keys()) == {'compression', 'bits_per_node', 'bits_per_edge', 'avg_locality'} assert set(stats['indegree'].keys()) == {'min', 'max', 'avg'} assert set(stats['outdegree'].keys()) == {'min', 'max', 'avg'} assert stats['counts']['nodes'] == 21 assert stats['counts']['edges'] == 23 assert isinstance(stats['ratios']['compression'], float) assert isinstance(stats['ratios']['bits_per_node'], float) assert isinstance(stats['ratios']['bits_per_edge'], float) assert isinstance(stats['ratios']['avg_locality'], float) assert stats['indegree']['min'] == 0 assert stats['indegree']['max'] == 3 assert isinstance(stats['indegree']['avg'], float) assert stats['outdegree']['min'] == 0 assert stats['outdegree']['max'] == 3 assert isinstance(stats['outdegree']['avg'], float) def test_leaves(graph_client): actual = list(graph_client.leaves( 'swh:1:ori:0000000000000000000000000000000000000021' )) expected = [ 'swh:1:cnt:0000000000000000000000000000000000000001', 'swh:1:cnt:0000000000000000000000000000000000000004', 'swh:1:cnt:0000000000000000000000000000000000000005', 'swh:1:cnt:0000000000000000000000000000000000000007' ] assert set(actual) == set(expected) def test_neighbors(graph_client): actual = list(graph_client.neighbors( 'swh:1:rev:0000000000000000000000000000000000000009', direction='backward' )) expected = [ 'swh:1:snp:0000000000000000000000000000000000000020', 'swh:1:rel:0000000000000000000000000000000000000010', 'swh:1:rev:0000000000000000000000000000000000000013' ] assert set(actual) == set(expected) def test_visit_nodes(graph_client): actual = list(graph_client.visit_nodes( 'swh:1:rel:0000000000000000000000000000000000000010', edges='rel:rev,rev:rev' )) expected = [ 'swh:1:rel:0000000000000000000000000000000000000010', 'swh:1:rev:0000000000000000000000000000000000000009', 'swh:1:rev:0000000000000000000000000000000000000003' ] assert set(actual) == set(expected) def test_visit_paths(graph_client): actual = list(graph_client.visit_paths( 'swh:1:snp:0000000000000000000000000000000000000020', edges='snp:*,rev:*')) actual = [tuple(path) for path in actual] expected = [ ( 'swh:1:snp:0000000000000000000000000000000000000020', 'swh:1:rev:0000000000000000000000000000000000000009', 'swh:1:rev:0000000000000000000000000000000000000003', 'swh:1:dir:0000000000000000000000000000000000000002' ), ( 'swh:1:snp:0000000000000000000000000000000000000020', 'swh:1:rev:0000000000000000000000000000000000000009', 'swh:1:dir:0000000000000000000000000000000000000008' ), ( 'swh:1:snp:0000000000000000000000000000000000000020', 'swh:1:rel:0000000000000000000000000000000000000010' ) ] assert set(actual) == set(expected) def test_walk(graph_client): actual = list(graph_client.walk( 'swh:1:dir:0000000000000000000000000000000000000016', 'rel', edges='dir:dir,dir:rev,rev:*', direction='backward', traversal='bfs' )) expected = [ 'swh:1:dir:0000000000000000000000000000000000000016', 'swh:1:dir:0000000000000000000000000000000000000017', 'swh:1:rev:0000000000000000000000000000000000000018', 'swh:1:rel:0000000000000000000000000000000000000019' ] assert set(actual) == set(expected) def test_count(graph_client): - print(graph_client) actual = graph_client.count_leaves( 'swh:1:ori:0000000000000000000000000000000000000021' ) assert actual == 4 actual = graph_client.count_visit_nodes( 'swh:1:rel:0000000000000000000000000000000000000010', edges='rel:rev,rev:rev' ) assert actual == 3 actual = graph_client.count_neighbors( 'swh:1:rev:0000000000000000000000000000000000000009', direction='backward' ) assert actual == 3 diff --git a/swh/graph/tests/test_cli.py b/swh/graph/tests/test_cli.py index 7e24e0e..522a060 100644 --- a/swh/graph/tests/test_cli.py +++ b/swh/graph/tests/test_cli.py @@ -1,50 +1,47 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest from pathlib import Path from tempfile import TemporaryDirectory from typing import Dict from click.testing import CliRunner -import swh.graph - from swh.graph import cli def read_properties(properties_fname) -> Dict[str, str]: """read a Java .properties file""" properties = {} with open(properties_fname) as f: for line in f: if line.startswith('#'): continue (key, value) = line.rstrip().split('=', maxsplit=1) properties[key] = value return properties class TestCompress(unittest.TestCase): - DATA_DIR = Path(swh.graph.__file__).parent.parent.parent \ - / 'tests' / 'dataset' + DATA_DIR = Path(__file__).parents[0] / 'dataset' def setUp(self): self.runner = CliRunner() def test_pipeline(self): """run full compression pipeline""" with TemporaryDirectory(suffix='.swh-graph-test') as tmpdir: result = self.runner.invoke( cli.compress, ['--graph', self.DATA_DIR / 'example', '--outdir', tmpdir]) self.assertEqual(result.exit_code, 0) properties = read_properties(Path(tmpdir) / 'example.properties') self.assertEqual(int(properties['nodes']), 21) self.assertEqual(int(properties['arcs']), 23)