test: Add a sub-module for testing grammar

hangxingliu · Apr 28, 2024 · 2ae7b89 · 2ae7b89
1 parent 38918ef
commit 2ae7b89
Show file tree

Hide file tree

Showing 3 changed files with 109 additions and 0 deletions.
diff --git a/test/grammar/index.js b/test/grammar/index.js
@@ -0,0 +1,73 @@
+//@ts-check
+/// <reference types="node" />
+const fs = require("fs");
+const path = require("path");
+const vsctm = require("vscode-textmate");
+const oniguruma = require("vscode-oniguruma");
+
+const dim = (str) => `\x1b[2m${str}\x1b[22m`;
+const cyan = (str) => `\x1b[36m${str}\x1b[39m`;
+
+const grammar = {
+    scopeName: "source.mkosi",
+    file: path.resolve(__dirname, "../../src/syntax/mkosi.tmLanguage"),
+    // testFile: path.resolve(__dirname, "../../test/samples/mkosi/mkosi-tools/mkosi.conf"),
+    testInput: [
+        '[Host]',
+        '@Incremental=yes',
+        'KernelCommandLineExtra=systemd.crash_shell=yes',
+        '                      systemd.log_level=debug'
+    ].join('\n'),
+};
+
+const wasmBin = fs.readFileSync(path.join(__dirname, "./node_modules/vscode-oniguruma/release/onig.wasm")).buffer;
+const vscodeOnigurumaLib = oniguruma.loadWASM(wasmBin).then(() => {
+    return {
+        createOnigScanner(patterns) {
+            return new oniguruma.OnigScanner(patterns);
+        },
+        createOnigString(s) {
+            return new oniguruma.OnigString(s);
+        },
+    };
+});
+
+// Create a registry that can create a grammar from a scope name.
+const registry = new vsctm.Registry({
+    onigLib: vscodeOnigurumaLib,
+    loadGrammar: async (scopeName) => {
+        if (scopeName === grammar.scopeName) {
+            // https://github.com/textmate/javascript.tmbundle/blob/master/Syntaxes/JavaScript.plist
+            const grammarXML = fs.readFileSync(grammar.file, "utf-8");
+            return vsctm.parseRawGrammar(grammarXML);
+        }
+        console.log(`Unknown scope name: ${scopeName}`);
+        return null;
+    },
+});
+
+// Load the JavaScript grammar and any other grammars included by it async.
+registry.loadGrammar(grammar.scopeName).then((g) => {
+    if (!g) return;
+
+    const text = grammar.testFile ? fs.readFileSync(grammar.testFile, "utf-8") : grammar.testInput;
+    const lines = text.split(/\r?\n/);
+
+    let ruleStack = vsctm.INITIAL;
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const lineTokens = g.tokenizeLine(line, ruleStack);
+        console.log(`\nline             "${dim(line)}"`);
+
+        for (let j = 0; j < lineTokens.tokens.length; j++) {
+            const token = lineTokens.tokens[j];
+            const scopes = token.scopes.filter((it) => it !== grammar.scopeName);
+
+            const range = `[${token.startIndex}, ${token.endIndex})`.padEnd(8, " ");
+            let log = ` - token${range} "${dim(line.substring(token.startIndex, token.endIndex))}" `;
+            if (scopes.length > 0) log += `scopes: ${scopes.map(cyan).join(", ")}`;
+            console.log(log);
+        }
+        ruleStack = lineTokens.ruleStack;
+    }
+});
diff --git a/test/grammar/package.json b/test/grammar/package.json
@@ -0,0 +1,11 @@
+{
+    "name": "@hangxingliu/test-textmate-grammar",
+    "version": "1.0.0",
+    "private": true,
+    "author": "hangxingliu",
+    "devDependencies": {
+        "@types/node": "^20.12.7",
+        "vscode-oniguruma": "^2.0.1",
+        "vscode-textmate": "^9.0.0"
+    }
+}
diff --git a/test/grammar/yarn.lock b/test/grammar/yarn.lock
@@ -0,0 +1,25 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@types/node@^20.12.7":
+  version "20.12.7"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.7.tgz#04080362fa3dd6c5822061aa3124f5c152cff384"
+  integrity sha512-wq0cICSkRLVaf3UGLMGItu/PtdY7oaXaI/RVU+xliKVOtRna3PRY57ZDfztpDL0n11vfymMUnXv8QwYCO7L1wg==
+  dependencies:
+    undici-types "~5.26.4"
+
+undici-types@~5.26.4:
+  version "5.26.5"
+  resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
+  integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==
+
+vscode-oniguruma@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-2.0.1.tgz#1196a343635ff8d42eb880e325b5f4e70731c02f"
+  integrity sha512-poJU8iHIWnC3vgphJnrLZyI3YdqRlR27xzqDmpPXYzA93R4Gk8z7T6oqDzDoHjoikA2aS82crdXFkjELCdJsjQ==
+
+vscode-textmate@^9.0.0:
+  version "9.0.0"
+  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-9.0.0.tgz#313c6c8792b0507aef35aeb81b6b370b37c44d6c"
+  integrity sha512-Cl65diFGxz7gpwbav10HqiY/eVYTO1sjQpmRmV991Bj7wAoOAjGQ97PpQcXorDE2Uc4hnGWLY17xme+5t6MlSg==