diff --git a/Makefile.local b/Makefile.local index a0f180b..d7eacdc 100644 --- a/Makefile.local +++ b/Makefile.local @@ -1,11 +1,11 @@ -POM_PATH=java/server/pom.xml +POM_PATH=java/pom.xml java: mvn -f $(POM_PATH) compile assembly:single java-%: mvn -f $(POM_PATH) $* clean-java: java-clean .PHONY: java clean-java diff --git a/docs/Makefile.local b/docs/Makefile.local index 49362c8..0cf14f3 100644 --- a/docs/Makefile.local +++ b/docs/Makefile.local @@ -1,22 +1,22 @@ -JAVA_DIR = ../java/server +JAVA_DIR = ../java sphinx/html: images sphinx/clean: clean-images clean-javadoc assets: images javadoc images: make -C images/ clean-images: make -C images/ clean javadoc: mvn -f $(JAVA_DIR)/pom.xml javadoc:javadoc clean-javadoc: mvn -f $(JAVA_DIR)/pom.xml clean .PHONY: images clean-images javadoc clean-javadoc # Local Variables: # mode: makefile # End: diff --git a/java/server/.clang-format b/java/.clang-format similarity index 100% rename from java/server/.clang-format rename to java/.clang-format diff --git a/java/server/.gitignore b/java/.gitignore similarity index 100% rename from java/server/.gitignore rename to java/.gitignore diff --git a/java/server/AUTHORS b/java/AUTHORS similarity index 100% rename from java/server/AUTHORS rename to java/AUTHORS diff --git a/java/server/LICENSE b/java/LICENSE similarity index 100% rename from java/server/LICENSE rename to java/LICENSE diff --git a/java/server/README.md b/java/README.md similarity index 100% rename from java/server/README.md rename to java/README.md diff --git a/java/server/pom.xml b/java/pom.xml similarity index 100% rename from java/server/pom.xml rename to java/pom.xml diff --git a/java/server/src/main/java/org/softwareheritage/graph/AllowedEdges.java b/java/src/main/java/org/softwareheritage/graph/AllowedEdges.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/AllowedEdges.java rename to java/src/main/java/org/softwareheritage/graph/AllowedEdges.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/App.java b/java/src/main/java/org/softwareheritage/graph/App.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/App.java rename to java/src/main/java/org/softwareheritage/graph/App.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/Endpoint.java b/java/src/main/java/org/softwareheritage/graph/Endpoint.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/Endpoint.java rename to java/src/main/java/org/softwareheritage/graph/Endpoint.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/Entry.java b/java/src/main/java/org/softwareheritage/graph/Entry.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/Entry.java rename to java/src/main/java/org/softwareheritage/graph/Entry.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/Graph.java b/java/src/main/java/org/softwareheritage/graph/Graph.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/Graph.java rename to java/src/main/java/org/softwareheritage/graph/Graph.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/Neighbors.java b/java/src/main/java/org/softwareheritage/graph/Neighbors.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/Neighbors.java rename to java/src/main/java/org/softwareheritage/graph/Neighbors.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/Node.java b/java/src/main/java/org/softwareheritage/graph/Node.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/Node.java rename to java/src/main/java/org/softwareheritage/graph/Node.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/SwhPID.java b/java/src/main/java/org/softwareheritage/graph/SwhPID.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/SwhPID.java rename to java/src/main/java/org/softwareheritage/graph/SwhPID.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/SwhPath.java b/java/src/main/java/org/softwareheritage/graph/SwhPath.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/SwhPath.java rename to java/src/main/java/org/softwareheritage/graph/SwhPath.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/algo/NodeIdConsumer.java b/java/src/main/java/org/softwareheritage/graph/algo/NodeIdConsumer.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/algo/NodeIdConsumer.java rename to java/src/main/java/org/softwareheritage/graph/algo/NodeIdConsumer.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/algo/NodeIdsConsumer.java b/java/src/main/java/org/softwareheritage/graph/algo/NodeIdsConsumer.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/algo/NodeIdsConsumer.java rename to java/src/main/java/org/softwareheritage/graph/algo/NodeIdsConsumer.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/algo/PathConsumer.java b/java/src/main/java/org/softwareheritage/graph/algo/PathConsumer.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/algo/PathConsumer.java rename to java/src/main/java/org/softwareheritage/graph/algo/PathConsumer.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/algo/Stats.java b/java/src/main/java/org/softwareheritage/graph/algo/Stats.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/algo/Stats.java rename to java/src/main/java/org/softwareheritage/graph/algo/Stats.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/algo/Traversal.java b/java/src/main/java/org/softwareheritage/graph/algo/Traversal.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/algo/Traversal.java rename to java/src/main/java/org/softwareheritage/graph/algo/Traversal.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/backend/MapBuilder.java b/java/src/main/java/org/softwareheritage/graph/backend/MapBuilder.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/backend/MapBuilder.java rename to java/src/main/java/org/softwareheritage/graph/backend/MapBuilder.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/backend/MapFile.java b/java/src/main/java/org/softwareheritage/graph/backend/MapFile.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/backend/MapFile.java rename to java/src/main/java/org/softwareheritage/graph/backend/MapFile.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/backend/NodeIdMap.java b/java/src/main/java/org/softwareheritage/graph/backend/NodeIdMap.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/backend/NodeIdMap.java rename to java/src/main/java/org/softwareheritage/graph/backend/NodeIdMap.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/backend/NodeTypesMap.java b/java/src/main/java/org/softwareheritage/graph/backend/NodeTypesMap.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/backend/NodeTypesMap.java rename to java/src/main/java/org/softwareheritage/graph/backend/NodeTypesMap.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/backend/Pp.java b/java/src/main/java/org/softwareheritage/graph/backend/Pp.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/backend/Pp.java rename to java/src/main/java/org/softwareheritage/graph/backend/Pp.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/benchmark/AccessEdge.java b/java/src/main/java/org/softwareheritage/graph/benchmark/AccessEdge.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/benchmark/AccessEdge.java rename to java/src/main/java/org/softwareheritage/graph/benchmark/AccessEdge.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/benchmark/BFS.java b/java/src/main/java/org/softwareheritage/graph/benchmark/BFS.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/benchmark/BFS.java rename to java/src/main/java/org/softwareheritage/graph/benchmark/BFS.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/benchmark/Benchmark.java b/java/src/main/java/org/softwareheritage/graph/benchmark/Benchmark.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/benchmark/Benchmark.java rename to java/src/main/java/org/softwareheritage/graph/benchmark/Benchmark.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/benchmark/Browsing.java b/java/src/main/java/org/softwareheritage/graph/benchmark/Browsing.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/benchmark/Browsing.java rename to java/src/main/java/org/softwareheritage/graph/benchmark/Browsing.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/benchmark/ForkCC.java b/java/src/main/java/org/softwareheritage/graph/benchmark/ForkCC.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/benchmark/ForkCC.java rename to java/src/main/java/org/softwareheritage/graph/benchmark/ForkCC.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/benchmark/GenDistribution.java b/java/src/main/java/org/softwareheritage/graph/benchmark/GenDistribution.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/benchmark/GenDistribution.java rename to java/src/main/java/org/softwareheritage/graph/benchmark/GenDistribution.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/benchmark/Provenance.java b/java/src/main/java/org/softwareheritage/graph/benchmark/Provenance.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/benchmark/Provenance.java rename to java/src/main/java/org/softwareheritage/graph/benchmark/Provenance.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/benchmark/Vault.java b/java/src/main/java/org/softwareheritage/graph/benchmark/Vault.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/benchmark/Vault.java rename to java/src/main/java/org/softwareheritage/graph/benchmark/Vault.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/benchmark/utils/Random.java b/java/src/main/java/org/softwareheritage/graph/benchmark/utils/Random.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/benchmark/utils/Random.java rename to java/src/main/java/org/softwareheritage/graph/benchmark/utils/Random.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/benchmark/utils/Statistics.java b/java/src/main/java/org/softwareheritage/graph/benchmark/utils/Statistics.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/benchmark/utils/Statistics.java rename to java/src/main/java/org/softwareheritage/graph/benchmark/utils/Statistics.java diff --git a/java/server/src/main/java/org/softwareheritage/graph/benchmark/utils/Timing.java b/java/src/main/java/org/softwareheritage/graph/benchmark/utils/Timing.java similarity index 100% rename from java/server/src/main/java/org/softwareheritage/graph/benchmark/utils/Timing.java rename to java/src/main/java/org/softwareheritage/graph/benchmark/utils/Timing.java diff --git a/java/server/src/test/java/org/softwareheritage/graph/AllowedEdgesTest.java b/java/src/test/java/org/softwareheritage/graph/AllowedEdgesTest.java similarity index 100% rename from java/server/src/test/java/org/softwareheritage/graph/AllowedEdgesTest.java rename to java/src/test/java/org/softwareheritage/graph/AllowedEdgesTest.java diff --git a/java/server/src/test/java/org/softwareheritage/graph/GraphTest.java b/java/src/test/java/org/softwareheritage/graph/GraphTest.java similarity index 100% rename from java/server/src/test/java/org/softwareheritage/graph/GraphTest.java rename to java/src/test/java/org/softwareheritage/graph/GraphTest.java diff --git a/java/server/src/test/java/org/softwareheritage/graph/LeavesTest.java b/java/src/test/java/org/softwareheritage/graph/LeavesTest.java similarity index 100% rename from java/server/src/test/java/org/softwareheritage/graph/LeavesTest.java rename to java/src/test/java/org/softwareheritage/graph/LeavesTest.java diff --git a/java/server/src/test/java/org/softwareheritage/graph/NeighborsTest.java b/java/src/test/java/org/softwareheritage/graph/NeighborsTest.java similarity index 100% rename from java/server/src/test/java/org/softwareheritage/graph/NeighborsTest.java rename to java/src/test/java/org/softwareheritage/graph/NeighborsTest.java diff --git a/java/server/src/test/java/org/softwareheritage/graph/VisitTest.java b/java/src/test/java/org/softwareheritage/graph/VisitTest.java similarity index 100% rename from java/server/src/test/java/org/softwareheritage/graph/VisitTest.java rename to java/src/test/java/org/softwareheritage/graph/VisitTest.java diff --git a/java/server/src/test/java/org/softwareheritage/graph/WalkTest.java b/java/src/test/java/org/softwareheritage/graph/WalkTest.java similarity index 100% rename from java/server/src/test/java/org/softwareheritage/graph/WalkTest.java rename to java/src/test/java/org/softwareheritage/graph/WalkTest.java diff --git a/setup.py b/setup.py index 751640d..4cab260 100755 --- a/setup.py +++ b/setup.py @@ -1,75 +1,75 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from setuptools import setup, find_packages from os import path from io import open from glob import glob here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = 'requirements-%s.txt' % name else: reqf = 'requirements.txt' requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith('#'): continue requirements.append(line) return requirements -JAR_PATHS = list(glob('java/server/target/swh-graph-*.jar')) +JAR_PATHS = list(glob('java/target/swh-graph-*.jar')) setup( name='swh.graph', description='Software Heritage graph service', long_description=long_description, long_description_content_type='text/markdown', author='Software Heritage developers', author_email='swh-devel@inria.fr', url='https://forge.softwareheritage.org/diffusion/DGRPH', packages=find_packages(), install_requires=parse_requirements() + parse_requirements('swh'), tests_require=parse_requirements('test'), setup_requires=['vcversioner'], extras_require={'testing': parse_requirements('test')}, vcversioner={}, include_package_data=True, data_files=[('share/swh-graph', JAR_PATHS)], entry_points=''' [console_scripts] swh-graph=swh.graph.cli:main [swh.cli.subcommands] graph=swh.graph.cli:cli ''', classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 3 - Alpha", ], project_urls={ 'Bug Reports': 'https://forge.softwareheritage.org/maniphest', 'Funding': 'https://www.softwareheritage.org/donate', 'Source': 'https://forge.softwareheritage.org/source/swh-graph', }, ) diff --git a/swh/graph/backend.py b/swh/graph/backend.py index 9dffc5c..aa11449 100644 --- a/swh/graph/backend.py +++ b/swh/graph/backend.py @@ -1,205 +1,205 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import asyncio import contextlib import io import os import pathlib import struct import subprocess import sys import tempfile from py4j.java_gateway import JavaGateway from swh.graph.pid import NodeToPidMap, PidToNodeMap from swh.model.identifiers import PID_TYPES BUF_SIZE = 64*1024 BIN_FMT = '>q' # 64 bit integer, big endian PATH_SEPARATOR_ID = -1 NODE2PID_EXT = 'node2pid.bin' PID2NODE_EXT = 'pid2node.bin' def find_graph_jar(): """find swh-graph.jar, containing the Java part of swh-graph look both in development directories and installed data (for in-production deployments who fecthed the JAR from pypi) """ swh_graph_root = pathlib.Path(__file__).parents[2] try_paths = [ - swh_graph_root / 'java/server/target/', + swh_graph_root / 'java/target/', pathlib.Path(sys.prefix) / 'share/swh-graph/', ] for path in try_paths: glob = list(path.glob('swh-graph-*.jar')) if glob: return str(glob[0]) raise RuntimeError("swh-graph-*.jar not found. Have you run `make java`?") def _get_pipe_stderr(): # Get stderr if possible, or pipe to stdout if running with Jupyter. try: sys.stderr.fileno() except io.UnsupportedOperation: return subprocess.STDOUT else: return sys.stderr class Backend: def __init__(self, graph_path): self.gateway = None self.entry = None self.graph_path = graph_path def __enter__(self): # TODO: make all of that configurable with sane defaults java_opts = [ '-Xmx200G', '-server', '-XX:PretenureSizeThreshold=512M', '-XX:MaxNewSize=4G', '-XX:+UseLargePages', '-XX:+UseTransparentHugePages', '-XX:+UseNUMA', '-XX:+UseTLAB', '-XX:+ResizeTLAB', ] self.gateway = JavaGateway.launch_gateway( java_path=None, javaopts=java_opts, classpath=find_graph_jar(), die_on_exit=True, redirect_stdout=sys.stdout, redirect_stderr=_get_pipe_stderr(), ) self.entry = self.gateway.jvm.org.softwareheritage.graph.Entry() self.entry.load_graph(self.graph_path) self.node2pid = NodeToPidMap(self.graph_path + '.' + NODE2PID_EXT) self.pid2node = PidToNodeMap(self.graph_path + '.' + PID2NODE_EXT) self.stream_proxy = JavaStreamProxy(self.entry) return self def __exit__(self, exc_type, exc_value, tb): self.gateway.shutdown() def stats(self): return self.entry.stats() def count(self, ttype, direction, edges_fmt, src): method = getattr(self.entry, 'count_' + ttype) return method(direction, edges_fmt, src) async def simple_traversal(self, ttype, direction, edges_fmt, src): assert ttype in ('leaves', 'neighbors', 'visit_nodes') method = getattr(self.stream_proxy, ttype) async for node_id in method(direction, edges_fmt, src): yield node_id async def walk(self, direction, edges_fmt, algo, src, dst): if dst in PID_TYPES: it = self.stream_proxy.walk_type(direction, edges_fmt, algo, src, dst) else: it = self.stream_proxy.walk(direction, edges_fmt, algo, src, dst) async for node_id in it: yield node_id async def visit_paths(self, direction, edges_fmt, src): path = [] async for node in self.stream_proxy.visit_paths( direction, edges_fmt, src): if node == PATH_SEPARATOR_ID: yield path path = [] else: path.append(node) class JavaStreamProxy: """A proxy class for the org.softwareheritage.graph.Entry Java class that takes care of the setup and teardown of the named-pipe FIFO communication between Python and Java. Initialize JavaStreamProxy using: proxy = JavaStreamProxy(swh_entry_class_instance) Then you can call an Entry method and iterate on the FIFO results like this: async for value in proxy.java_method(arg1, arg2): print(value) """ def __init__(self, entry): self.entry = entry async def read_node_ids(self, fname): loop = asyncio.get_event_loop() open_thread = loop.run_in_executor(None, open, fname, 'rb') # Since the open() call on the FIFO is blocking until it is also opened # on the Java side, we await it with a timeout in case there is an # exception that prevents the write-side open(). with (await asyncio.wait_for(open_thread, timeout=2)) as f: while True: data = await loop.run_in_executor(None, f.read, BUF_SIZE) if not data: break for data in struct.iter_unpack(BIN_FMT, data): yield data[0] class _HandlerWrapper: def __init__(self, handler): self._handler = handler def __getattr__(self, name): func = getattr(self._handler, name) async def java_call(*args, **kwargs): loop = asyncio.get_event_loop() await loop.run_in_executor(None, lambda: func(*args, **kwargs)) def java_task(*args, **kwargs): return asyncio.create_task(java_call(*args, **kwargs)) return java_task @contextlib.contextmanager def get_handler(self): with tempfile.TemporaryDirectory(prefix='swh-graph-') as tmpdirname: cli_fifo = os.path.join(tmpdirname, 'swh-graph.fifo') os.mkfifo(cli_fifo) reader = self.read_node_ids(cli_fifo) query_handler = self.entry.get_handler(cli_fifo) handler = self._HandlerWrapper(query_handler) yield (handler, reader) def __getattr__(self, name): async def java_call_iterator(*args, **kwargs): with self.get_handler() as (handler, reader): java_task = getattr(handler, name)(*args, **kwargs) try: async for value in reader: yield value except asyncio.TimeoutError: # If the read-side open() timeouts, an exception on the # Java side probably happened that prevented the # write-side open(). We propagate this exception here if # that is the case. task_exc = java_task.exception() if task_exc: raise task_exc raise await java_task return java_call_iterator diff --git a/tox.ini b/tox.ini index 742bf95..557eb85 100644 --- a/tox.ini +++ b/tox.ini @@ -1,30 +1,30 @@ [tox] envlist=flake8,mypy,py3 [testenv:py3] deps = .[testing] pytest-cov whitelist_externals = mvn sh commands = - sh -c 'if ! [ -d {envdir}/share/swh-graph ]; then mvn -f java/server/pom.xml compile assembly:single; mkdir {envdir}/share/swh-graph; cp java/server/target/*.jar {envdir}/share/swh-graph; fi' + sh -c 'if ! [ -d {envdir}/share/swh-graph ]; then mvn -f java/pom.xml compile assembly:single; mkdir {envdir}/share/swh-graph; cp java/target/*.jar {envdir}/share/swh-graph; fi' pytest --cov={envsitepackagesdir}/swh/graph \ {envsitepackagesdir}/swh/graph \ --cov-branch {posargs} [testenv:flake8] skip_install = true deps = flake8 commands = {envpython} -m flake8 [testenv:mypy] skip_install = true deps = .[testing] mypy commands = mypy swh