Implement semantic tokens

internal/fourslash/_scripts/convertFourslash.mts

@@ -85,13 +85,20 @@ function parseFileContent(filename: string, content: string): GoTest | undefined
     };
     for (const statement of statements) {
         const result = parseFourslashStatement(statement);
+        if (result === SKIP_STATEMENT) {
+            // Skip this statement but continue parsing
+            continue;
+        }
         if (!result) {
+            // Could not parse this statement - mark file as unparsed
             unparsedFiles.push(filename);
             return undefined;
         }
-        else {
-            goTest.commands.push(...result);
-        }
+        goTest.commands.push(...result);
+    }
+    // Skip tests that have no commands (e.g., only syntactic classifications)
+    if (goTest.commands.length === 0) {
+        return undefined;
     }
     return goTest;
 }
@@ -134,11 +141,22 @@ function getTestInput(content: string): string {
     return `\`${testInput.join("\n")}\``;
 }
 
+// Sentinel value to indicate a statement should be skipped but parsing should continue
+const SKIP_STATEMENT: unique symbol = Symbol("SKIP_STATEMENT");
+type SkipStatement = typeof SKIP_STATEMENT;
+
 /**
  * Parses a Strada fourslash statement and returns the corresponding Corsa commands.
- * @returns an array of commands if the statement is a valid fourslash command, or `false` if the statement could not be parsed.
+ * @returns an array of commands if the statement is a valid fourslash command,
+ *          SKIP_STATEMENT if the statement should be skipped but parsing should continue,
+ *          or `undefined` if the statement could not be parsed and the file should be marked as unparsed.
  */
-function parseFourslashStatement(statement: ts.Statement): Cmd[] | undefined {
+function parseFourslashStatement(statement: ts.Statement): Cmd[] | SkipStatement | undefined {
+    // Skip empty statements (bare semicolons)
+    if (ts.isEmptyStatement(statement)) {
+        return SKIP_STATEMENT;
+    }
+
     if (ts.isVariableStatement(statement)) {
         // variable declarations (for ranges and markers), e.g. `const range = test.ranges()[0];`
         return [];
@@ -210,6 +228,10 @@ function parseFourslashStatement(statement: ts.Statement): Cmd[] | undefined {
                 case "renameInfoSucceeded":
                 case "renameInfoFailed":
                     return parseRenameInfo(func.text, callExpression.arguments);
+                case "semanticClassificationsAre":
+                    return parseSemanticClassificationsAre(callExpression.arguments);
+                case "syntacticClassificationsAre":
+                    return SKIP_STATEMENT;
             }
         }
         // `goTo....`
@@ -1509,6 +1531,71 @@ function parseBaselineSmartSelection(args: ts.NodeArray<ts.Expression>): Cmd {
     };
 }
 
+function parseSemanticClassificationsAre(args: readonly ts.Expression[]): [VerifySemanticClassificationsCmd] | SkipStatement | undefined {
+    if (args.length < 1) {
+        console.error("semanticClassificationsAre requires at least a format argument");
+        return undefined;
+    }
+
+    const formatArg = args[0];
+    if (!ts.isStringLiteralLike(formatArg)) {
+        console.error("semanticClassificationsAre first argument must be a string literal");
+        return undefined;
+    }
+
+    const format = formatArg.text;
+
+    // Only handle "2020" format for semantic tokens
+    if (format !== "2020") {
+        // Skip other formats like "original" - return sentinel to continue parsing
+        return SKIP_STATEMENT;
+    }
+
+    const tokens: Array<{ type: string; text: string; }> = [];
+
+    // Parse the classification tokens (c2.semanticToken("type", "text"))
+    for (let i = 1; i < args.length; i++) {
+        const arg = args[i];
+        if (!ts.isCallExpression(arg)) {
+            console.error(`Expected call expression for token at index ${i}`);
+            return undefined;
+        }
+
+        if (!ts.isPropertyAccessExpression(arg.expression) || arg.expression.name.text !== "semanticToken") {
+            console.error(`Expected semanticToken call at index ${i}`);
+            return undefined;
+        }
+
+        if (arg.arguments.length < 2) {
+            console.error(`semanticToken requires 2 arguments at index ${i}`);
+            return undefined;
+        }
+
+        const typeArg = arg.arguments[0];
+        const textArg = arg.arguments[1];
+
+        if (!ts.isStringLiteralLike(typeArg) || !ts.isStringLiteralLike(textArg)) {
+            console.error(`semanticToken arguments must be string literals at index ${i}`);
+            return undefined;
+        }
+
+        // Map TypeScript's internal "member" type to LSP's "method" type
+        let tokenType = typeArg.text;
+        tokenType = tokenType.replace(/\bmember\b/g, "method");
+
+        tokens.push({
+            type: tokenType,
+            text: textArg.text,
+        });
+    }
+
+    return [{
+        kind: "verifySemanticClassifications",
+        format,
+        tokens,
+    }];
+}
+
 function parseKind(expr: ts.Expression): string | undefined {
     if (!ts.isStringLiteral(expr)) {
         console.error(`Expected string literal for kind, got ${expr.getText()}`);
@@ -1725,6 +1812,12 @@ interface VerifyRenameInfoCmd {
     preferences: string;
 }
 
+interface VerifySemanticClassificationsCmd {
+    kind: "verifySemanticClassifications";
+    format: string;
+    tokens: Array<{ type: string; text: string; }>;
+}
+
 type Cmd =
     | VerifyCompletionsCmd
     | VerifyApplyCodeActionFromCompletionCmd
@@ -1739,7 +1832,8 @@ type Cmd =
     | VerifyQuickInfoCmd
     | VerifyBaselineRenameCmd
     | VerifyRenameInfoCmd
-    | VerifyBaselineInlayHintsCmd;
+    | VerifyBaselineInlayHintsCmd
+    | VerifySemanticClassificationsCmd;
 
 function generateVerifyCompletions({ marker, args, isNewIdentifierLocation, andApplyCodeActionArgs }: VerifyCompletionsCmd): string {
     let expectedList: string;
@@ -1840,6 +1934,14 @@ function generateBaselineInlayHints({ span, preferences }: VerifyBaselineInlayHi
     return `f.VerifyBaselineInlayHints(t, ${span}, ${preferences})`;
 }
 
+function generateSemanticClassifications({ format, tokens }: VerifySemanticClassificationsCmd): string {
+    const tokensStr = tokens.map(t => `{Type: ${getGoStringLiteral(t.type)}, Text: ${getGoStringLiteral(t.text)}}`).join(",\n\t\t");
+    const maybeComma = tokens.length > 0 ? "," : "";
+    return `f.VerifySemanticTokens(t, []fourslash.SemanticToken{
+		${tokensStr}${maybeComma}
+	})`;
+}
+
 function generateCmd(cmd: Cmd): string {
     switch (cmd.kind) {
         case "verifyCompletions":
@@ -1878,6 +1980,8 @@ function generateCmd(cmd: Cmd): string {
             return `f.VerifyRenameFailed(t, ${cmd.preferences})`;
         case "verifyBaselineInlayHints":
             return generateBaselineInlayHints(cmd);
+        case "verifySemanticClassifications":
+            return generateSemanticClassifications(cmd);
         default:
             let neverCommand: never = cmd;
             throw new Error(`Unknown command kind: ${neverCommand as Cmd["kind"]}`);

internal/fourslash/_scripts/manualTests.txt

@@ -4,4 +4,5 @@ completionsSelfDeclaring1
 completionsWithDeprecatedTag4
 renameDefaultKeyword
 renameForDefaultExport01
+semanticModernClassificationFunctions
 tsxCompletion12

internal/fourslash/fourslash.go

@@ -50,6 +50,10 @@ type FourslashTest struct {
 	lastKnownMarkerName  *string
 	activeFilename       string
 	selectionEnd         *lsproto.Position
+
+	// Semantic token configuration
+	semanticTokenTypes     []string
+	semanticTokenModifiers []string
 }
 
 type scriptInfo struct {
@@ -183,15 +187,17 @@ func NewFourslash(t *testing.T, capabilities *lsproto.ClientCapabilities, conten
 	})
 
 	f := &FourslashTest{
-		server:          server,
-		in:              inputWriter,
-		out:             outputReader,
-		testData:        &testData,
-		userPreferences: lsutil.NewDefaultUserPreferences(), // !!! parse default preferences for fourslash case?
-		vfs:             fs,
-		scriptInfos:     scriptInfos,
-		converters:      converters,
-		baselines:       make(map[string]*strings.Builder),
+		server:                 server,
+		in:                     inputWriter,
+		out:                    outputReader,
+		testData:               &testData,
+		userPreferences:        lsutil.NewDefaultUserPreferences(), // !!! parse default preferences for fourslash case?
+		vfs:                    fs,
+		scriptInfos:            scriptInfos,
+		converters:             converters,
+		baselines:              make(map[string]*strings.Builder),
+		semanticTokenTypes:     defaultSemanticTokenTypes(),
+		semanticTokenModifiers: defaultSemanticTokenModifiers(),
 	}
 
 	// !!! temporary; remove when we have `handleDidChangeConfiguration`/implicit project config support
@@ -231,6 +237,50 @@ func (f *FourslashTest) initialize(t *testing.T, capabilities *lsproto.ClientCap
 	sendNotification(t, f, lsproto.InitializedInfo, &lsproto.InitializedParams{})
 }
 
+func defaultSemanticTokenTypes() []string {
+	return []string{
+		string(lsproto.SemanticTokenTypesnamespace),
+		string(lsproto.SemanticTokenTypesclass),
+		string(lsproto.SemanticTokenTypesenum),
+		string(lsproto.SemanticTokenTypesinterface),
+		string(lsproto.SemanticTokenTypesstruct),
+		string(lsproto.SemanticTokenTypestypeParameter),
+		string(lsproto.SemanticTokenTypestype),
+		string(lsproto.SemanticTokenTypesparameter),
+		string(lsproto.SemanticTokenTypesvariable),
+		string(lsproto.SemanticTokenTypesproperty),
+		string(lsproto.SemanticTokenTypesenumMember),
+		string(lsproto.SemanticTokenTypesdecorator),
+		string(lsproto.SemanticTokenTypesevent),
+		string(lsproto.SemanticTokenTypesfunction),
+		string(lsproto.SemanticTokenTypesmethod),
+		string(lsproto.SemanticTokenTypesmacro),
+		string(lsproto.SemanticTokenTypeslabel),
+		string(lsproto.SemanticTokenTypescomment),
+		string(lsproto.SemanticTokenTypesstring),
+		string(lsproto.SemanticTokenTypeskeyword),
+		string(lsproto.SemanticTokenTypesnumber),
+		string(lsproto.SemanticTokenTypesregexp),
+		string(lsproto.SemanticTokenTypesoperator),
+	}
+}
+
+func defaultSemanticTokenModifiers() []string {
+	return []string{
+		string(lsproto.SemanticTokenModifiersdeclaration),
+		string(lsproto.SemanticTokenModifiersdefinition),
+		string(lsproto.SemanticTokenModifiersreadonly),
+		string(lsproto.SemanticTokenModifiersstatic),
+		string(lsproto.SemanticTokenModifiersdeprecated),
+		string(lsproto.SemanticTokenModifiersabstract),
+		string(lsproto.SemanticTokenModifiersasync),
+		string(lsproto.SemanticTokenModifiersmodification),
+		string(lsproto.SemanticTokenModifiersdocumentation),
+		string(lsproto.SemanticTokenModifiersdefaultLibrary),
+		"local",
+	}
+}
+
 var (
 	ptrTrue                       = ptrTo(true)
 	defaultCompletionCapabilities = &lsproto.CompletionClientCapabilities{
@@ -293,6 +343,18 @@ func getCapabilitiesWithDefaults(capabilities *lsproto.ClientCapabilities) *lspr
 			},
 		}
 	}
+	if capabilitiesWithDefaults.TextDocument.SemanticTokens == nil {
+		capabilitiesWithDefaults.TextDocument.SemanticTokens = &lsproto.SemanticTokensClientCapabilities{
+			Requests: &lsproto.ClientSemanticTokensRequestOptions{
+				Full: &lsproto.BooleanOrClientSemanticTokensRequestFullDelta{
+					Boolean: ptrTrue,
+				},
+			},
+			TokenTypes:     defaultSemanticTokenTypes(),
+			TokenModifiers: defaultSemanticTokenModifiers(),
+			Formats:        []lsproto.TokenFormat{lsproto.TokenFormatRelative},
+		}
+	}
 	if capabilitiesWithDefaults.Workspace == nil {
 		capabilitiesWithDefaults.Workspace = &lsproto.WorkspaceClientCapabilities{}
 	}