diff --git a/package.json b/package.json index 523b762..e8b665e 100644 --- a/package.json +++ b/package.json @@ -1,36 +1,34 @@ { "name": "swh-search-query-language-parser", "version": "1.0.0", "description": "Parser for Software Heritage archive search query language", "scripts": { - "generate": "cd query_language && tree-sitter generate --no-bindings && echo 'Generated parser files '", - "dev": "yarn generate && cd query_language && tree-sitter parse sample_query", - "test": "yarn generate && cd query_language && tree-sitter test", - "build-so": "yarn generate && cd query_language && python3 build.py", - "build-wasm": "yarn generate && cd query_language && tree-sitter build-wasm . && mv tree-sitter-swh_search_ql.wasm swh_ql.wasm", + "generate": "cd swh/search/query_language && tree-sitter generate --no-bindings && echo 'Generated parser files '", + "dev": "yarn generate && cd swh/search/query_language && tree-sitter parse sample_query", + "test": "yarn generate && cd swh/search/query_language && tree-sitter test", "build": "echo 'use `pip3 install .` or `pip3 wheel .` instead.'", - "repl": "yarn generate && cd query_language && tree-sitter build-wasm && tree-sitter playground" + "repl": "yarn generate && cd swh/search/query_language && tree-sitter build-wasm && tree-sitter playground" }, "repository": { "type": "git", "url": "https://forge.softwareheritage.org/source/swh-search.git" }, "keywords": [ "swh", "Software Heritage", "treesitter", "parser", "custom", "search", "query", "language" ], "author": "The Software Heritage developers", "license": "GPL-3.0-only", "dependencies": { "nan": "^2.14.2" }, "devDependencies": { "tree-sitter-cli": "^0.20.0" } } diff --git a/query_language/build.py b/query_language/build.py deleted file mode 100644 index 62c3de2..0000000 --- a/query_language/build.py +++ /dev/null @@ -1,3 +0,0 @@ -from tree_sitter import Language - -Language.build_library("swh_ql.so", ["."]) diff --git a/setup.py b/setup.py index 52de412..bf3108d 100755 --- a/setup.py +++ b/setup.py @@ -1,201 +1,182 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from distutils.cmd import Command +from distutils.command.build import build from io import open import os import shutil import subprocess -import sys from setuptools import find_packages, setup -from setuptools.command.build_py import build_py from setuptools.command.sdist import sdist here = os.path.abspath(os.path.dirname(__file__)) # Get the long description from the README file with open(os.path.join(here, "README.md"), encoding="utf-8") as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = "requirements-%s.txt" % name else: reqf = "requirements.txt" requirements = [] if not os.path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith("#"): continue requirements.append(line) return requirements yarn = os.environ.get("YARN", "yarn") class TSCommand(Command): user_options = [] def initialize_options(self): pass def finalize_options(self): pass class TSInstallCommand(TSCommand): description = "Installs node_modules related to query language" def run(self): subprocess.run([yarn, "install"], check=True) -class TSGenerateCommand(TSCommand): - description = "Generates parser related files from grammar.js" +class TSBuildSoCommand(TSCommand): + description = "Builds swh_ql.so" + + def initialize_options(self): + self.build_lib = None + super().initialize_options() + + def finalize_options(self): + self.set_undefined_options("build", ("build_lib", "build_lib")) + super().finalize_options() def run(self): - subprocess.run([yarn, "generate"], check=True) + self.run_command("ts_install") + ql_dir = os.path.join(self.build_lib, "swh/search/query_language") + if not os.path.exists(os.path.join(ql_dir, "src/parser.c")): + generate_parser(ql_dir) -class TSBuildSoCommand(TSCommand): - description = "Builds swh_ql.so" + static_dir = os.path.join(self.build_lib, "swh/search/static") + os.makedirs(static_dir, exist_ok=True) - def run(self): - # setup_requires changes sys.path so the build dependencies - # can be imported even though they are in a temporary - # directory (usually `.eggs`). We need to pass this updated sys.path to - # 'yarn build-so', as it invokes a Python script that needs to import - # tree_sitter - env = {**os.environ, "PYTHONPATH": os.pathsep.join(sys.path)} - subprocess.run([yarn, "build-so"], check=True, env=env) + # This import cannot be toplevel, as setuptools installs it after the script + # starts running + from tree_sitter import Language + + Language.build_library(os.path.join(static_dir, "swh_ql.so"), [ql_dir]) print("swh_ql.so file generated") class TSBuildWasmCommand(TSCommand): description = "Builds swh_ql.wasm" def run(self): subprocess.run([yarn, "build-wasm"], check=True) print("swh_ql.wasm file generated") class TSBuildCommand(TSCommand): description = "Builds swh_ql.so and swh_ql.wasm" def run(self): self.run_command("ts_build_so") - self.run_command("ts_build_wasm") - - -class TSBuildExportCommand(TSCommand): - description = "Builds swh_ql.so and swh_ql.wasm and exports them to static/" - - def initialize_options(self): - self.build_lib = None - super().initialize_options() - - def finalize_options(self): - self.set_undefined_options("build", ("build_lib", "build_lib")) - super().finalize_options() - - def run(self): - self.run_command("ts_install") - self.run_command("ts_build") - - print("static files generated. copying them to package dir") - os.makedirs(os.path.join(self.build_lib, "swh/search/static"), exist_ok=True) - shutil.copyfile( - "query_language/swh_ql.so", - os.path.join(self.build_lib, "swh/search/static/swh_ql.so"), - ) - shutil.copyfile( - "query_language/swh_ql.wasm", - os.path.join(self.build_lib, "swh/search/static/swh_ql.wasm"), - ) -class custom_build(build_py): +class custom_build(build): def run(self): super().run() if not self.dry_run: - self.run_command("ts_build_export") + self.run_command("ts_build") class custom_sdist(sdist): def make_release_tree(self, base_dir, files): super().make_release_tree(base_dir, files) - # TODO: build the .c file and .wasm but not .so, because it's architecture- - # dependent, and shouldn't be in a sdist (aka *source* distribution) + + dist_ql_path = os.path.join(base_dir, "swh/search/query_language") + if not self.dry_run: self.run_command("ts_install") - self.run_command("ts_build") - print("static files generated. copying them to package dir") - os.makedirs(os.path.join(base_dir, "swh/search/static"), exist_ok=True) - shutil.copyfile( - "query_language/swh_ql.so", - os.path.join(base_dir, "swh/search/static/swh_ql.so"), - ) - shutil.copyfile( - "query_language/swh_ql.wasm", - os.path.join(base_dir, "swh/search/static/swh_ql.wasm"), - ) + generate_parser(dist_ql_path) + + +def generate_parser(dest_path): + # FIXME: setuptools should copy this itself... + print("Copying parser files") + if os.path.exists(dest_path): + shutil.rmtree(dest_path) + shutil.copytree("swh/search/query_language", dest_path) + + print("Getting path") + path = subprocess.check_output([yarn, "bin"]).decode().strip() + env = {**os.environ, "PATH": os.pathsep.join([path, os.environ["PATH"]])} + print("Generating") + subprocess.run(["tree-sitter", "generate", "--no-bindings"], cwd=dest_path, env=env) setup( name="swh.search", description="Software Heritage search service", long_description=long_description, long_description_content_type="text/markdown", python_requires=">=3.7", author="Software Heritage developers", author_email="swh-devel@inria.fr", url="https://forge.softwareheritage.org/diffusion/DSEA", packages=find_packages(), # packages's modules install_requires=parse_requirements() + parse_requirements("swh"), tests_require=parse_requirements("test"), entry_points=""" [swh.cli.subcommands] search=swh.search.cli """, setup_requires=["setuptools-scm", "tree-sitter==0.19.0"], use_scm_version=True, extras_require={"testing": parse_requirements("test")}, include_package_data=True, classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 3 - Alpha", ], project_urls={ "Bug Reports": "https://forge.softwareheritage.org/maniphest", "Funding": "https://www.softwareheritage.org/donate", "Source": "https://forge.softwareheritage.org/source/swh-search", "Documentation": "https://docs.softwareheritage.org/devel/swh-search/", }, cmdclass={ - "build_py": custom_build, + "build": custom_build, "sdist": custom_sdist, "ts_install": TSInstallCommand, - "ts_generate": TSGenerateCommand, "ts_build_so": TSBuildSoCommand, - "ts_build_wasm": TSBuildWasmCommand, "ts_build": TSBuildCommand, - "ts_build_export": TSBuildExportCommand, }, zip_safe=False, ) diff --git a/query_language/.gitignore b/swh/search/query_language/.gitignore similarity index 100% rename from query_language/.gitignore rename to swh/search/query_language/.gitignore diff --git a/query_language/grammar.js b/swh/search/query_language/grammar.js similarity index 100% rename from query_language/grammar.js rename to swh/search/query_language/grammar.js diff --git a/query_language/sample_query b/swh/search/query_language/sample_query similarity index 100% rename from query_language/sample_query rename to swh/search/query_language/sample_query diff --git a/query_language/test/corpus/combinations.txt b/swh/search/query_language/test/corpus/combinations.txt similarity index 100% rename from query_language/test/corpus/combinations.txt rename to swh/search/query_language/test/corpus/combinations.txt diff --git a/query_language/tokens.js b/swh/search/query_language/tokens.js similarity index 100% rename from query_language/tokens.js rename to swh/search/query_language/tokens.js