build test grammar infra

Signed-off-by: Qingpeng Li <qingpeng9802@gmail.com>

Author: qingpeng9802

.eslintrc.json

@@ -7,7 +7,7 @@
     "root": true,
     "parser": "@typescript-eslint/parser",
     "parserOptions": {
-        "project": "./tsconfig.json",
+        "project": ["tsconfig.json", "syntaxes/scripts/tsconfig.json"],
         "ecmaVersion": 2022,
         "sourceType": "module"
     },
@@ -48,6 +48,7 @@
         "eol-last": "warn",
         "one-var": [ "warn", "never" ],
         "new-parens": "warn",
+        "template-curly-spacing": ["warn"],
         "prefer-const": "warn",
         "prefer-arrow-callback": "warn",
         "prefer-rest-params": "warn",
@@ -297,43 +298,50 @@
                 "zones": [
                     // layer 0
                     {
-                        "target": "./src/web/common",
-                        "from": "./src/web",
-                        "message": "[layer 0] ./src/web/common"
+                        "target": "src/web/common",
+                        "from": "src/web",
+                        "message": "[layer 0] src/web/common"
                     },
                     {
-                        "target": "./src/web/doc",
-                        "from": "./src/web",
-                        "except": [ "./cl_data", "./common" ],
-                        "message": "[layer 0] ./src/web/doc"
+                        "target": "src/web/doc",
+                        "from": "src/web",
+                        "except": [ "cl_data", "common" ],
+                        "message": "[layer 0] src/web/doc"
                     },
                     {
-                        "target": "./src/web/collect_info",
-                        "from": "./src/web",
-                        "except": [ "./collect_info", "./common" ],
-                        "message": "[layer 0] ./src/web/collect_info"
+                        "target": "src/web/collect_info",
+                        "from": "src/web",
+                        "except": [ "collect_info", "common" ],
+                        "message": "[layer 0] src/web/collect_info"
                     },
                     // layer 1
                     {
-                        "target": "./src/web/builders",
-                        "from": [ "./src/web/entry", "./src/web/extension.ts", "./src/web/provider_interface" ],
-                        "message": "[layer 1] ./src/web/builders"
+                        "target": "src/web/builders",
+                        "from": [ "src/web/entry", "src/web/extension.ts", "src/web/provider_interface" ],
+                        "message": "[layer 1] src/web/builders"
                     },
                     // layer 2
                     {
-                        "target": "./src/web/provider_interface",
-                        "from": [ "./src/web/entry", "./src/web/extension.ts" ],
-                        "message": "[layer 2] ./src/web/provider_interface"
+                        "target": "src/web/provider_interface",
+                        "from": [ "src/web/entry", "src/web/extension.ts" ],
+                        "message": "[layer 2] src/web/provider_interface"
                     },
                     // layer 3
                     {
-                        "target": "./src/web/entry",
-                        "from": "./src/web/extension.ts",
-                        "message": "[layer 3] ./src/web/entry"
+                        "target": "src/web/entry",
+                        "from": "src/web/extension.ts",
+                        "message": "[layer 3] src/web/entry"
                     }
                 ]
             }
-        ]
+        ],
+
+        "node/no-unsupported-features/node-builtins": ["error", {
+            "version": ">=20.8.0"
+        }],
+        "node/no-unpublished-import": ["error", {
+            "allowModules": ["vscode-oniguruma", "vscode-textmate", "js-yaml"]
+        }]
     },
 
     "settings": {
@@ -350,6 +358,7 @@
         }
     },
     "ignorePatterns": [
-        "**/*.d.ts"
+        "**/*.d.ts",
+        "syntaxes/scripts/gen_record.mjs"
     ]
 }

.github/ISSUE_TEMPLATE/textmate_syntax_related.md

@@ -0,0 +1,33 @@
+---
+name: Textmate-syntax related
+about: Bug or improvement of Textmate-syntax
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+<!-- Troubleshooting -->
+Please help confirm if it is a purely Textmate grammar issue:
+1. set `"commonLisp.StaticAnalysis.enabled": false` in this extension's built-in settings (under `Common Lisp` tab) or set `"editor.semanticHighlighting.enabled": false` in VS Code's settings.
+2. reload VS Code.
+3. check if you still see the bug and describe your findings.
+
+-   VS Code Version: *You can find it by* [*how-do-i-find-the-version*](https://code.visualstudio.com/docs/supporting/FAQ#_how-do-i-find-the-version)
+-   Desktop Electron Version or Browser version: *You can find it by* [*how-do-i-find-the-version*](https://code.visualstudio.com/docs/supporting/FAQ#_how-do-i-find-the-version)
+-   OS Version: *which OS and its version*
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Code**
+```lsp
+;; Please include a code snippet that demonstrates the issue
+
+```
+
+**Current Behavior Screenshot**
+
+
+**Expected Behavior Screenshot**
+

.github/workflows/build.yml

@@ -50,5 +50,5 @@ jobs:
       - name: build with esbuild
         run: npm run esbuildp
 
-  #test:
-  #  uses: ./.github/workflows/test.yml
+  test:
+    uses: ./.github/workflows/test.yml

.github/workflows/test.yml

@@ -34,5 +34,8 @@ jobs:
       - name: Build Grammar from yaml to json
         run: npm run bg
 
-      - name: Run headless test
-        run: xvfb-run -a npm test
+      - name: Test Grammar
+        run: npm run testg
+
+      #- name: Headless Test for the Web Extension
+      #  run: xvfb-run -a npm test

.github/workflows/vsce_publish.yml

@@ -11,8 +11,8 @@ on:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  #test:
-  #  uses: ./.github/workflows/test.yml
+  test:
+    uses: ./.github/workflows/test.yml
 
   publish:
     name: publish

.github/workflows/vsce_publish_pre.yml

@@ -8,8 +8,8 @@ on:
         required: true
 
 jobs:
-  #test:
-  #  uses: ./.github/workflows/test.yml
+  test:
+    uses: ./.github/workflows/test.yml
 
   publish_pre:
     name: publish pre-release

.gitignore

@@ -5,3 +5,4 @@ node_modules
 *.vsix
 *.log
 .parcel-cache
+syntaxes/test/generated/*

.vscode/launch.json

@@ -22,7 +22,6 @@
                 "!**/node_modules/**"
             ],
             "outFiles": ["${workspaceFolder}/dist/web/**/*.js"],
-            "skipFiles": ["<node_internals>/**"],
             //"preLaunchTask": "npm: esbuildc"
         },
         {
@@ -44,7 +43,6 @@
                 "!**/node_modules/**"
             ],
 			"outFiles": ["${workspaceFolder}/dist/web/**/*.js"],
-            "skipFiles": ["<node_internals>/**"],
 			//"preLaunchTask": "npm: esbuildc"
 		},
         {

CONTRIBUTING.md

@@ -109,7 +109,8 @@ Run `find ./src -name '*.ts' | xargs wc -l` to check the physical lines of code
 ## Declarative Language Features  
 
 ### Syntax Highlight
-We have two syntax files in `./syntaxes`, and we use `./scripts/build_grammar.mjs` to convert them into `json` format:
+#### Introduction
+We have two syntax files in `./syntaxes`, and we use `./syntaxes/scripts/build_grammar.mjs` to convert them into `json` format:
 - `commonlisp.yaml` is the main syntax of Common Lisp.
 - `cl_codeblock.yaml` is the [injection grammar](https://code.visualstudio.com/api/language-extensions/syntax-highlight-guide#injection-grammars) for Markdown.
 
@@ -121,13 +122,21 @@ If you need to modify the syntax, here are some helpful materials:
 [regex101](https://regex101.com/),
 [Common Lisp HyperSpec](http://www.lispworks.com/documentation/HyperSpec/Front/).
 
+#### Workflow
 Mastering the TextMate grammars is very difficult. If you are a beginner, you can try to start with a small work sample and then work on it incrementally.  
 The recommended workflow is:  
 1. `git add`
 2. construct a regex and verify the regex in [regex101](https://regex101.com/)
 3. add the regex to the syntax
 4. convert the syntax into `json` and test the syntax in real world
 
+#### Maintain grammar health
+1. After you modified the grammar, run `npm run bg` to rebuild the grammar.  
+2. Then, run `npm run testg` to test the grammar with the fixtures in `syntaxes/test/baselines`.
+    > You may optionally run `npx tsc -p syntaxes/scripts/tsconfig.json` to generate `gen_record.mjs`.
+3. Check if it is the test result you expected. Run `npm run diff` to find the differences between the baselines and generated.
+4. If the changes are what you expect, run `npm run accept` to accept the new baselines.
+
 ### Snippet
 In `./snippets`.
 See [Snippet Guide](https://code.visualstudio.com/api/language-extensions/snippet-guide) and [creating-your-own-snippets](https://code.visualstudio.com/docs/editor/userdefinedsnippets#_creating-your-own-snippets).