add feature: optionally de-color quoted parts

revert experimental feature: `updatePairMatchFirstWord()`

Signed-off-by: Qingpeng Li <qingpeng9802@gmail.com>

Author: qingpeng9802

package-lock.json

@@ -202,6 +202,11 @@
                         "Respond to all the selected words."
                     ]
                 },
+                "commonLisp.DocumentSemanticTokensProvider.NotColorQuoted": {
+                    "type": "boolean",
+                    "default": false,
+                    "markdownDescription": "If true, any `quote` will NOT be colored."
+                },
                 "commonLisp.DocumentSemanticTokensProvider.ExcludedRanges": {
                     "type": "string",
                     "default": "comment and string",
@@ -303,24 +308,24 @@
     },
     "devDependencies": {
         "@types/js-yaml": "^4.0.9",
-        "@types/node": "^20.9.0",
+        "@types/node": "^20.9.2",
         "@types/vscode": "1.63.0",
         "@types/webpack-env": "^1.18.4",
-        "@typescript-eslint/eslint-plugin": "^6.10.0",
+        "@typescript-eslint/eslint-plugin": "^6.11.0",
         "assert": "^2.1.0",
-        "dependency-cruiser": "^15.2.0",
-        "esbuild": "^0.19.5",
-        "eslint": "^8.53.0",
+        "dependency-cruiser": "^15.3.0",
+        "esbuild": "^0.19.6",
+        "eslint": "^8.54.0",
         "eslint-import-resolver-typescript": "^3.6.1",
         "eslint-plugin-import": "^2.29.0",
         "eslint-plugin-node": "^11.1.0",
         "eslint-plugin-unused-imports": "^3.0.0",
         "fs": "^0.0.1-security",
         "js-yaml": "^4.1.0",
         "process": "^0.11.10",
-        "ts-loader": "^9.5.0",
+        "ts-loader": "^9.5.1",
         "typescript": "^5.2.2",
-        "unimported": "^1.30.0",
+        "unimported": "^1.31.1",
         "webpack": "^5.89.0",
         "webpack-cli": "^5.1.4",
         "copyfiles": "^2.4.1",

package.json

@@ -6,7 +6,7 @@ import type { DocSymbolInfo } from '../DocSymbolInfo';
 import { getScopedSameNameWordsExcludeItself } from '../builders_util';
 
 import { _encodeTokenType, _encodeTokenModifiers, vscodeKindToTokenType } from './token_util';
-import { overrideQuote, updateLoop, updatePairMatchFirstWord } from './update_util';
+import { overrideQuote, updateLoop } from './update_util';
 
 function genAllPossibleWord(
   currDocSymbolInfo: DocSymbolInfo
@@ -61,32 +61,19 @@ function getTokenDict(
   const excludedRanges: [number, number][] =
     currDocSymbolInfo.docRes.getExcludedRangesForDocumentSemanticTokensProvider(buildingConfig);
 
-  // colored position collector
-  // {index, [encodeTokenType, encodeTokenModifiers]}
-  // Note that the `index` should be no-translated result, that is,
-  // `index` should be the 1st character of current string.
-  // For example, `sb-c::instrument-consing` should be the index of `s` instead of `i`.
-  let coloredPosMap: Map<number, [number, number]> = new Map<number, [number, number]>();
-
   // overlap in order!
-  updateTokenDict(currDocSymbolInfo, excludedRanges, needColorDict, 'local', tokensBuilder, coloredPosMap);
-
-  // Due to the restrictions below, we cannot make this feature optional
-  // https://github.com/microsoft/vscode/issues/580
-  // https://github.com/microsoft/vscode/issues/68647
-  updatePairMatchFirstWord(currDocSymbolInfo, coloredPosMap, tokensBuilder);
-
-  coloredPosMap = new Map<number, [number, number]>();
+  updateTokenDict(currDocSymbolInfo, excludedRanges, needColorDict, 'local', tokensBuilder);
+  updateTokenDict(currDocSymbolInfo, excludedRanges, needColorDict, 'global', tokensBuilder);
 
-  updateTokenDict(currDocSymbolInfo, excludedRanges, needColorDict, 'global', tokensBuilder, coloredPosMap);
-
-  updateLoop(currDocSymbolInfo, excludedRanges, tokensBuilder, coloredPosMap);
-  updatePairMatchFirstWord(currDocSymbolInfo, coloredPosMap, tokensBuilder);
-  coloredPosMap = new Map<number, [number, number]>();
+  updateLoop(currDocSymbolInfo, excludedRanges, tokensBuilder);
 
   // 1. tried to de-color all single-quoted parts
-  // vscode semantic highlighting does not support multi-line token, does not work
-  overrideQuote(currDocSymbolInfo, tokensBuilder);
+  // vscode semantic highlighting does not support multi-line tokens
+  // so we split the multi-line tokens into multiple single-line tokens manually.
+  // This may have a negative impact on performance.
+  if (buildingConfig.get('commonLisp.DocumentSemanticTokensProvider.NotColorQuoted') === true) {
+    overrideQuote(currDocSymbolInfo, tokensBuilder);
+  }
   //
   // 2. tried to de-color all no-formatted strings
   // not sure if it can cover all cases or not
@@ -100,7 +87,6 @@ function updateTokenDict(
   needColorDict: Map<string, [number, number][]>,
   updateScope: 'global' | 'local',
   tokensBuilder: vscode.SemanticTokensBuilder,
-  coloredPosMap: Map<number, [number, number]>,
 ) {
   const isGlobal = updateScope === 'global';
   const d = isGlobal ? currDocSymbolInfo.globalDef : currDocSymbolInfo.allLocal;
@@ -125,11 +111,7 @@ function updateTokenDict(
 
       // color def itself
       const startPos = item.startPos;
-      const encoded = setParsedToken(tokensBuilder, item, startPos, isGlobal);
-      if (encoded !== undefined) {
-        coloredPosMap.set(item.numRange[0], encoded);
-      }
-
+      setParsedToken(tokensBuilder, item, startPos, isGlobal);
       // color its scope
       const scopedSameNameWords = getScopedSameNameWordsExcludeItself(item, needColorDict, currDocSymbolInfo);
       for (const rang of scopedSameNameWords) {
@@ -138,10 +120,7 @@ function updateTokenDict(
         }
 
         const startPos = currDocSymbolInfo.document.positionAt(rang[0]);
-        const encoded = setParsedToken(tokensBuilder, item, startPos);
-        if (encoded !== undefined) {
-          coloredPosMap.set(rang[0], encoded);
-        }
+        setParsedToken(tokensBuilder, item, startPos);
       }
 
     }
@@ -155,7 +134,7 @@ function setParsedToken(
   item: SymbolInfo,
   startPos: vscode.Position,
   isGlobal = true
-): [number, number] | undefined {
+) {
   const nameStr = item.name;
   const packagePrefixIndScope = nameStr.indexOf(':');
   let len = nameStr.length;
@@ -168,7 +147,7 @@ function setParsedToken(
         isGlobal &&
         (subItemName.startsWith('+') && subItemName.endsWith('+')) ||
         (subItemName.startsWith('*') && subItemName.endsWith('*'))) {
-        return undefined;
+        return;
       }
     }
     startPos = startPos.translate(0, packagePrefixIndScope);
@@ -194,8 +173,6 @@ function setParsedToken(
   );
   //const key = `${item.name}|${startPos.line},${startPos.character},${item.name.length}`;
   //console.log(key);
-
-  return [encodedTT, encodedTMs];
 }
 
 function buildSemanticTokens(

src/web/builders/semantic_tokens_builder/semantic_tokens_builder.ts

@@ -10,7 +10,6 @@ function updateLoop(
   currDocSymbolInfo: DocSymbolInfo,
   excludedRanges: [number, number][],
   tokensBuilder: vscode.SemanticTokensBuilder,
-  coloredPosMap: Map<number, [number, number]>,
 ) {
   const document = currDocSymbolInfo.document;
   const text = document.getText();
@@ -62,7 +61,6 @@ function updateLoop(
         startPos.line, startPos.character, word.length,
         encodedTT, encodedTMs
       );
-      coloredPosMap.set(rindex, [encodedTT, encodedTMs]);
       //const key = `loop ${word}|${startPos.line},${startPos.character},${word.length}`;
       //console.log(key);
     }
@@ -145,6 +143,9 @@ function overrideNotFormattedString(
   }
 }
 
+// Due to the restrictions below, we cannot make this feature optional
+// https://github.com/microsoft/vscode/issues/580
+// https://github.com/microsoft/vscode/issues/68647
 function updatePairMatchFirstWord(
   currDocSymbolInfo: DocSymbolInfo,
   coloredPosMap: Map<number, [number, number]>,

src/web/builders/semantic_tokens_builder/update_util.ts

@@ -23,6 +23,8 @@ class WorkspaceConfig {
     ['commonLisp.DefinitionProvider.ExcludedRanges', ExcludeRanges.None],
     ['commonLisp.DefinitionProvider.BackQuoteFilter.enabled', true],
 
+    ['commonLisp.DocumentSemanticTokensProvider.NotColorQuoted', false],
+
     ['commonLisp.providers.CompletionItemProviders.user.enabled', true],
     ['commonLisp.providers.CompletionItemProviders.original.enabled', true],
     ['commonLisp.providers.CompletionItemProviders.loop.enabled', true],

src/web/entry/WorkspaceConfig.ts

@@ -36,6 +36,8 @@ class StructuredInfo {
     ['commonLisp.DefinitionProvider.ExcludedRanges', ExcludeRanges.None],
     ['commonLisp.ReferenceProvider.ExcludedRanges', ExcludeRanges.CommentString],
     ['commonLisp.DocumentSemanticTokensProvider.ExcludedRanges', ExcludeRanges.CommentString],
+
+    ['commonLisp.DocumentSemanticTokensProvider.NotColorQuoted', false],
   ]);
 
   // dirty flag indicates that the document has been changed,