From 068659c90f2f11e3c2ea707091f377dae6060227 Mon Sep 17 00:00:00 2001 From: Ethan Leba Date: Thu, 14 Apr 2022 08:41:47 -0700 Subject: [PATCH] Add local parser building functionality --- CHANGELOG.md | 3 + bin/test | 2 + core/Cargo.lock | 646 ++++++++++++++++++++++++++++++++++++++++++++- core/Cargo.toml | 4 +- core/src/build.rs | 28 ++ core/src/lib.rs | 1 + core/tsc.el | 22 +- tests/tsc-build.el | 74 ++++++ 8 files changed, 774 insertions(+), 6 deletions(-) create mode 100644 core/src/build.rs create mode 100644 tests/tsc-build.el diff --git a/CHANGELOG.md b/CHANGELOG.md index c31290fd..16384574 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +## [0.19.0] - 2022-04-27 +- Add `tsc-build-parser-from-source` for building tree sitter grammars locally + ## [0.18.0] - 2022-02-12 - Added APIs to traverse the syntax tree: `tsc-traverse-do`, `tsc-traverse-mapc`, `tsc-traverse-iter`. The traversal is depth-first pre-order. - Improved syntax tree rendering's performance in `tree-sitter-debug`. diff --git a/bin/test b/bin/test index afc63451..f82de41b 100755 --- a/bin/test +++ b/bin/test @@ -14,6 +14,8 @@ if [[ $* == "watch" ]]; then else if [[ $* == "integ" ]]; then test_mod="tsc-dyn-get-tests.el" + elif [[ $* == "build" ]]; then + test_mod="tsc-build.el" elif [[ $* == "bench" ]]; then test_mod="tree-sitter-bench.el" else diff --git a/core/Cargo.lock b/core/Cargo.lock index 7565a59c..b6232655 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -11,12 +11,56 @@ dependencies = [ "memchr", ] +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + [[package]] name = "anyhow" version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b26702f315f53b6071259e15dd9d64528213b44d61de1ec926eca7715d62203" +[[package]] +name = "ascii" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf56136a5198c7b01a49e3afcbef6cf84597273d298f54432926024107b0109" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" + [[package]] name = "cc" version = "1.0.72" @@ -29,6 +73,39 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "winapi", +] + +[[package]] +name = "chunked_transfer" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim 0.8.0", + "textwrap", + "unicode-width", + "vec_map", +] + [[package]] name = "ctor" version = "0.1.21" @@ -59,7 +136,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.9.3", "syn", ] @@ -74,6 +151,38 @@ dependencies = [ "syn", ] +[[package]] +name = "difference" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" + +[[package]] +name = "dirs" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + [[package]] name = "emacs" version = "0.18.0" @@ -103,12 +212,14 @@ dependencies = [ [[package]] name = "emacs-tree-sitter" -version = "0.18.0" +version = "0.19.0" dependencies = [ "emacs", "libloading", "once_cell", "tree-sitter", + "tree-sitter-cli", + "tree-sitter-loader", ] [[package]] @@ -123,12 +234,111 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "getrandom" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "html-escape" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e7479fa1ef38eb49fb6a42c426be515df2d063f06cb8efd3e50af073dbc26c" +dependencies = [ + "utf8-width", +] + [[package]] name = "ident_case" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + +[[package]] +name = "js-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec647867e2bf0772e28c8bcde4f0d19a9216916e890543b5a03ed8ef27b8f259" + [[package]] name = "libloading" version = "0.7.2" @@ -139,18 +349,58 @@ dependencies = [ "winapi", ] +[[package]] +name = "log" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + [[package]] name = "memchr" version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + [[package]] name = "proc-macro2" version = "1.0.33" @@ -169,6 +419,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "redox_syscall" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall", + "thiserror", +] + [[package]] name = "regex" version = "1.5.4" @@ -186,13 +456,34 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "semver", + "semver 0.9.0", +] + +[[package]] +name = "ryu" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", ] [[package]] @@ -204,12 +495,62 @@ dependencies = [ "semver-parser", ] +[[package]] +name = "semver" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" + [[package]] name = "semver-parser" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "serde" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "smallbitvec" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75ce4f9dc4a41b4c3476cc925f1efb11b66df373a8fde5d4b8915fa91b5d995e" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + [[package]] name = "strsim" version = "0.9.3" @@ -227,6 +568,15 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "thiserror" version = "1.0.30" @@ -247,6 +597,43 @@ dependencies = [ "syn", ] +[[package]] +name = "tiny_http" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce51b50006056f590c9b7c3808c3bd70f0d1101666629713866c227d6e58d39" +dependencies = [ + "ascii", + "chrono", + "chunked_transfer", + "log", + "url", +] + +[[package]] +name = "tinyvec" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + [[package]] name = "tree-sitter" version = "0.20.0" @@ -256,12 +643,256 @@ dependencies = [ "regex", ] +[[package]] +name = "tree-sitter-cli" +version = "0.20.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab14de5f4c74f5115872ede7ec7089cb0208762f61cac0b4899faf415d04dcd" +dependencies = [ + "ansi_term", + "anyhow", + "atty", + "clap", + "difference", + "dirs", + "glob", + "html-escape", + "indexmap", + "lazy_static", + "log", + "regex", + "regex-syntax", + "rustc-hash", + "semver 1.0.7", + "serde", + "serde_json", + "smallbitvec", + "tiny_http", + "toml", + "tree-sitter", + "tree-sitter-config", + "tree-sitter-highlight", + "tree-sitter-loader", + "tree-sitter-tags", + "walkdir", + "webbrowser", + "which", +] + +[[package]] +name = "tree-sitter-config" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5fec4cb27f052ead2246631b332dba0cb6af9a54ce012badee59c4b0ded5e03" +dependencies = [ + "anyhow", + "dirs", + "serde", + "serde_json", +] + +[[package]] +name = "tree-sitter-highlight" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "042342584c5a7a0b833d9fc4e2bdab3f9868ddc6c4b339a1e01451c6720868bc" +dependencies = [ + "regex", + "thiserror", + "tree-sitter", +] + +[[package]] +name = "tree-sitter-loader" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0b17eef4833c7c139abed66d562dfa23228e97e647597baf246fd56c21bbfaf" +dependencies = [ + "anyhow", + "cc", + "dirs", + "libloading", + "once_cell", + "regex", + "serde", + "serde_json", + "tree-sitter", + "tree-sitter-highlight", + "tree-sitter-tags", +] + +[[package]] +name = "tree-sitter-tags" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccb3f1376219530a37a809751ecf65aa35fd8b9c1c4ab6d4faf5f6a9eeda2c05" +dependencies = [ + "memchr", + "regex", + "thiserror", + "tree-sitter", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" + +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + [[package]] name = "unicode-xid" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "utf8-width" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5190c9442dcdaf0ddd50f37420417d219ae5261bbf5db120d0f9bab996c9cba1" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "wasm-bindgen" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" + +[[package]] +name = "web-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webbrowser" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecad156490d6b620308ed411cfee90d280b3cbd13e189ea0d3fada8acc89158a" +dependencies = [ + "web-sys", + "widestring", + "winapi", +] + +[[package]] +name = "which" +version = "4.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" +dependencies = [ + "either", + "lazy_static", + "libc", +] + +[[package]] +name = "widestring" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" + [[package]] name = "winapi" version = "0.3.9" @@ -278,6 +909,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/core/Cargo.toml b/core/Cargo.toml index a5e038e0..e7ab0c6e 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "emacs-tree-sitter" -version = "0.18.0" +version = "0.19.0" authors = ["Tuấn-Anh Nguyễn "] edition = "2018" publish = false @@ -12,6 +12,8 @@ crate-type = ["cdylib"] [dependencies] emacs = "0.18" +tree-sitter-cli = "0.20" +tree-sitter-loader = "0.20" libloading = "0.7.0" tree-sitter = "0.20.0" once_cell = "1.7.2" diff --git a/core/src/build.rs b/core/src/build.rs new file mode 100644 index 00000000..b73131c8 --- /dev/null +++ b/core/src/build.rs @@ -0,0 +1,28 @@ +use std::path::{Path, PathBuf}; +use tree_sitter_cli::generate; + +use emacs::{defun, Result, Value}; + +const DEFAULT_GENERATE_ABI_VERSION: usize = 13; + +/// Build the grammar at SRC_PATH and place into DST_PATH. +/// +/// Not intended to be used directly, see `tsc-build-parser-from-source'. +#[defun] +fn _build_parser_from_source(src_path: String, dst_path: String, generate: Value) -> Result<()> { + let abi_version = DEFAULT_GENERATE_ABI_VERSION; + let parsed_repo_path = PathBuf::from(&src_path); + if generate.is_not_nil() { + generate::generate_parser_in_directory( + &parsed_repo_path, + None, + abi_version, + false, + None, + )?; + } + let loader = tree_sitter_loader::Loader::with_parser_lib_path(Path::new(&dst_path).to_path_buf()); + let source_path = parsed_repo_path.join("src"); + loader.load_language_at_path(source_path.as_path(), source_path.as_path())?; + Ok(()) +} diff --git a/core/src/lib.rs b/core/src/lib.rs index f0e7207d..36e618b3 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -10,6 +10,7 @@ mod tree; mod node; mod cursor; mod query; +mod build; emacs::plugin_is_GPL_compatible! {} diff --git a/core/tsc.el b/core/tsc.el index 5dfd5892..35d879ee 100644 --- a/core/tsc.el +++ b/core/tsc.el @@ -6,7 +6,7 @@ ;; Jorge Javier Araya Navarro ;; Keywords: languages tools parsers dynamic-modules tree-sitter ;; Homepage: https://github.com/emacs-tree-sitter/elisp-tree-sitter -;; Version: 0.18.0 +;; Version: 0.19.0 ;; Package-Requires: ((emacs "25.1")) ;; SPDX-License-Identifier: MIT @@ -17,6 +17,7 @@ ;;; Code: +(require 'cl-lib) (require 'tsc-obsolete) (unless (functionp 'module-load) @@ -24,7 +25,7 @@ ;; Load the dynamic module at compile time as well, to satisfy the byte compiler. (eval-and-compile - (defconst tsc--dyn-version "0.18.0" + (defconst tsc--dyn-version "0.19.0" "Required version of the dynamic module `tsc-dyn'.") (require 'tsc-dyn-get) (tsc-dyn-get-ensure tsc--dyn-version)) @@ -455,5 +456,22 @@ If a step cannot be followed, signal a `tsc--invalid-node-step' error." (setq this new-node))) this)) +(cl-defun tsc-build-parser-from-source (src-path &key generate dst-path) + "Build the tree-sitter grammar at SRC-PATH. + +If GENERATE is non-nil, the C parser will be generated from the `grammar.js' +file in the repository root. In most cases, this is not necessary as the C +parser is typically included in the repository. + +DST-PATH specifies whether the build result should be placed. If +DST-PATH is unset, it will default to the value of +`tree-sitter-langs-grammar-dir' if bound. Otherwise, an error will be raised." + (if-let ((dst-path (or dst-path (bound-and-true-p tree-sitter-langs-grammar-dir)))) + (tsc--build-parser-from-source + (expand-file-name src-path) + (expand-file-name dst-path) + generate) + (user-error "No dst-path provided and `tree-sitter-langs-grammar-dir' is undefined"))) + (provide 'tsc) ;;; tsc.el ends here diff --git a/tests/tsc-build.el b/tests/tsc-build.el new file mode 100644 index 00000000..89245985 --- /dev/null +++ b/tests/tsc-build.el @@ -0,0 +1,74 @@ +;;; tsc-build.el --- Tests for tsc-dyn-get.el -*- lexical-binding: t; coding: utf-8 -*- + +;; Copyright (C) 2021 Tuấn-Anh Nguyễn +;; +;; Author: Tuấn-Anh Nguyễn +;; SPDX-License-Identifier: MIT + +;;; Commentary: + +;; Tests for the code that builds tree-sitter grammars into shared objects. + +(require 'async) +(require 'tsc) + +(eval-when-compile + (require 'subr-x)) + +(ert-deftest tsc-build:no-generate () + (let* ((test-dir (make-temp-file "tsc-build-test-" :directory)) + (repo-path (file-name-concat test-dir "tree-sitter-python")) + (shared-lib-path (file-name-concat test-dir "python.so"))) + (shell-command (format "git clone %s %s" + "https://github.com/tree-sitter/tree-sitter-python.git" + repo-path)) + + ;; Confirm shared object has been built + (should (not (file-exists-p shared-lib-path))) + (tsc-build-parser-from-source repo-path :dst-path test-dir) + (should (file-exists-p shared-lib-path)) + + ;; Confirm shared object is not corrupted and can be loaded properly + (async-sandbox + (lambda () + (require 'tree-sitter) + (tree-sitter-require 'my/python (file-name-concat test-dir "python") "tree_sitter_python"))))) + +(ert-deftest tsc-build:generate () + (let* ((test-dir (make-temp-file "tsc-build-test-" :directory)) + (repo-path (file-name-concat test-dir "tree-sitter-python")) + (shared-lib-path (file-name-concat test-dir "python.so"))) + (shell-command (format "git clone %s %s" + "https://github.com/tree-sitter/tree-sitter-python.git" + repo-path)) + + ;; Delete pre-existing parser file. If the parser was not regenerated, the + ;; compilation will fail. + (delete-file (file-name-concat repo-path "src" "parser.c")) + + ;; Confirm shared object has been built + (should (not (file-exists-p shared-lib-path))) + (tsc-build-parser-from-source + repo-path + :dst-path test-dir + :generate t) + (should (file-exists-p shared-lib-path)) + + ;; Confirm shared object is not corrupted and can be loaded properly + (async-sandbox + (lambda () + (require 'tree-sitter) + (tree-sitter-require 'my/python (file-name-concat test-dir "python") "tree_sitter_python"))))) + +(ert-deftest tsc-build:wrong-directory () + (should-error + (tsc-build-parser-from-source + (temporary-file-directory) + :generate t + :dst-path (temporary-file-directory)) + :type 'rust-error)) + +;;; Code: +;; Local Variables: +;; no-byte-compile: t +;; End: