diff --git a/package.json b/package.json --- a/package.json +++ b/package.json @@ -3,13 +3,11 @@ "version": "1.0.0", "description": "Parser for Software Heritage archive search query language", "scripts": { - "generate": "cd query_language && tree-sitter generate --no-bindings && echo 'Generated parser files '", - "dev": "yarn generate && cd query_language && tree-sitter parse sample_query", - "test": "yarn generate && cd query_language && tree-sitter test", - "build-so": "yarn generate && cd query_language && python3 build.py", - "build-wasm": "yarn generate && cd query_language && tree-sitter build-wasm . && mv tree-sitter-swh_search_ql.wasm swh_ql.wasm", + "generate": "cd swh/search/query_language && tree-sitter generate --no-bindings && echo 'Generated parser files '", + "dev": "yarn generate && cd swh/search/query_language && tree-sitter parse sample_query", + "test": "yarn generate && cd swh/search/query_language && tree-sitter test", "build": "echo 'use `pip3 install .` or `pip3 wheel .` instead.'", - "repl": "yarn generate && cd query_language && tree-sitter build-wasm && tree-sitter playground" + "repl": "yarn generate && cd swh/search/query_language && tree-sitter build-wasm && tree-sitter playground" }, "repository": { "type": "git", diff --git a/query_language/build.py b/query_language/build.py deleted file mode 100644 --- a/query_language/build.py +++ /dev/null @@ -1,3 +0,0 @@ -from tree_sitter import Language - -Language.build_library("swh_ql.so", ["."]) diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -5,14 +5,13 @@ # See top-level LICENSE file for more information from distutils.cmd import Command +from distutils.command.build import build from io import open import os import shutil import subprocess -import sys from setuptools import find_packages, setup -from setuptools.command.build_py import build_py from setuptools.command.sdist import sdist here = os.path.abspath(os.path.dirname(__file__)) @@ -61,24 +60,32 @@ subprocess.run([yarn, "install"], check=True) -class TSGenerateCommand(TSCommand): - description = "Generates parser related files from grammar.js" +class TSBuildSoCommand(TSCommand): + description = "Builds swh_ql.so" + + def initialize_options(self): + self.build_lib = None + super().initialize_options() + + def finalize_options(self): + self.set_undefined_options("build", ("build_lib", "build_lib")) + super().finalize_options() def run(self): - subprocess.run([yarn, "generate"], check=True) + self.run_command("ts_install") + ql_dir = os.path.join(self.build_lib, "swh/search/query_language") + if not os.path.exists(os.path.join(ql_dir, "src/parser.c")): + generate_parser(ql_dir) -class TSBuildSoCommand(TSCommand): - description = "Builds swh_ql.so" + static_dir = os.path.join(self.build_lib, "swh/search/static") + os.makedirs(static_dir, exist_ok=True) - def run(self): - # setup_requires changes sys.path so the build dependencies - # can be imported even though they are in a temporary - # directory (usually `.eggs`). We need to pass this updated sys.path to - # 'yarn build-so', as it invokes a Python script that needs to import - # tree_sitter - env = {**os.environ, "PYTHONPATH": os.pathsep.join(sys.path)} - subprocess.run([yarn, "build-so"], check=True, env=env) + # This import cannot be toplevel, as setuptools installs it after the script + # starts running + from tree_sitter import Language + + Language.build_library(os.path.join(static_dir, "swh_ql.so"), [ql_dir]) print("swh_ql.so file generated") @@ -95,63 +102,40 @@ def run(self): self.run_command("ts_build_so") - self.run_command("ts_build_wasm") - - -class TSBuildExportCommand(TSCommand): - description = "Builds swh_ql.so and swh_ql.wasm and exports them to static/" - - def initialize_options(self): - self.build_lib = None - super().initialize_options() - - def finalize_options(self): - self.set_undefined_options("build", ("build_lib", "build_lib")) - super().finalize_options() - - def run(self): - self.run_command("ts_install") - self.run_command("ts_build") - - print("static files generated. copying them to package dir") - os.makedirs(os.path.join(self.build_lib, "swh/search/static"), exist_ok=True) - shutil.copyfile( - "query_language/swh_ql.so", - os.path.join(self.build_lib, "swh/search/static/swh_ql.so"), - ) - shutil.copyfile( - "query_language/swh_ql.wasm", - os.path.join(self.build_lib, "swh/search/static/swh_ql.wasm"), - ) -class custom_build(build_py): +class custom_build(build): def run(self): super().run() if not self.dry_run: - self.run_command("ts_build_export") + self.run_command("ts_build") class custom_sdist(sdist): def make_release_tree(self, base_dir, files): super().make_release_tree(base_dir, files) - # TODO: build the .c file and .wasm but not .so, because it's architecture- - # dependent, and shouldn't be in a sdist (aka *source* distribution) + + dist_ql_path = os.path.join(base_dir, "swh/search/query_language") + if not self.dry_run: self.run_command("ts_install") - self.run_command("ts_build") - print("static files generated. copying them to package dir") - os.makedirs(os.path.join(base_dir, "swh/search/static"), exist_ok=True) - shutil.copyfile( - "query_language/swh_ql.so", - os.path.join(base_dir, "swh/search/static/swh_ql.so"), - ) - shutil.copyfile( - "query_language/swh_ql.wasm", - os.path.join(base_dir, "swh/search/static/swh_ql.wasm"), - ) + generate_parser(dist_ql_path) + + +def generate_parser(dest_path): + # FIXME: setuptools should copy this itself... + print("Copying parser files") + if os.path.exists(dest_path): + shutil.rmtree(dest_path) + shutil.copytree("swh/search/query_language", dest_path) + + print("Getting path") + path = subprocess.check_output([yarn, "bin"]).decode().strip() + env = {**os.environ, "PATH": os.pathsep.join([path, os.environ["PATH"]])} + print("Generating") + subprocess.run(["tree-sitter", "generate", "--no-bindings"], cwd=dest_path, env=env) setup( @@ -188,14 +172,11 @@ "Documentation": "https://docs.softwareheritage.org/devel/swh-search/", }, cmdclass={ - "build_py": custom_build, + "build": custom_build, "sdist": custom_sdist, "ts_install": TSInstallCommand, - "ts_generate": TSGenerateCommand, "ts_build_so": TSBuildSoCommand, - "ts_build_wasm": TSBuildWasmCommand, "ts_build": TSBuildCommand, - "ts_build_export": TSBuildExportCommand, }, zip_safe=False, ) diff --git a/query_language/.gitignore b/swh/search/query_language/.gitignore rename from query_language/.gitignore rename to swh/search/query_language/.gitignore diff --git a/query_language/grammar.js b/swh/search/query_language/grammar.js rename from query_language/grammar.js rename to swh/search/query_language/grammar.js diff --git a/query_language/sample_query b/swh/search/query_language/sample_query rename from query_language/sample_query rename to swh/search/query_language/sample_query diff --git a/query_language/test/corpus/combinations.txt b/swh/search/query_language/test/corpus/combinations.txt rename from query_language/test/corpus/combinations.txt rename to swh/search/query_language/test/corpus/combinations.txt diff --git a/query_language/tokens.js b/swh/search/query_language/tokens.js rename from query_language/tokens.js rename to swh/search/query_language/tokens.js