[TypeSpec Validation] Add unit test framework (#26166)

ckairen · zman-ms · commit 5dba2220765b · 2024-01-09T15:30:55.000-08:00
diff --git a/eng/tools/TypeSpecValidation/.mocharc.yaml b/eng/tools/TypeSpecValidation/.mocharc.yaml
@@ -0,0 +1 @@
+spec: "dist/test/**/*.js"
diff --git a/eng/tools/TypeSpecValidation/cmd/tsv.js b/eng/tools/TypeSpecValidation/cmd/tsv.js
@@ -1,5 +1,5 @@
 #!/usr/bin/env node
 
-import { main } from "../dist/index.js"
+import { main } from "../dist/src/index.js"
 
 await main();
diff --git a/eng/tools/TypeSpecValidation/package.json b/eng/tools/TypeSpecValidation/package.json
@@ -14,12 +14,15 @@
     },
     "devDependencies": {
         "globby": "^13.1.1",
+        "mocha": "^10.2.0",
         "@types/debug": "^4.1.8",
+        "@types/mocha": "^10.0.2",
         "@types/node": "^18.16.18",
         "typescript": "~5.1.3"
     },
     "scripts": {
-        "postinstall": "tsc"
+        "postinstall": "tsc",
+        "test": "mocha --recursive --exit"
     },
     "engines": {
         "node": "^16.17.0 || ^18.3.0 || ^20.0.0"
diff --git a/eng/tools/TypeSpecValidation/src/index.ts b/eng/tools/TypeSpecValidation/src/index.ts
@@ -5,6 +5,7 @@ import { FormatRule } from "./rules/format.js";
 import { GitDiffRule } from "./rules/git-diff.js";
 import { LinterRulesetRule } from "./rules/linter-ruleset.js";
 import { NpmPrefixRule } from "./rules/npm-prefix.js";
+import { TsvRunnerHost } from "./tsv-runner-host.js";
 
 export async function main() {
   const args = process.argv.slice(2);
@@ -30,7 +31,7 @@ export async function main() {
   for (let i = 0; i < rules.length; i++) {
     const rule = rules[i];
     console.log("\nExecuting rule: " + rule.name);
-    const result = await rule.execute(folder);
+    const result = await rule.execute(TsvRunnerHost, folder);
     if (result.stdOutput) console.log(result.stdOutput);
     if (!result.success) {
       success = false;
diff --git a/eng/tools/TypeSpecValidation/src/rule.ts b/eng/tools/TypeSpecValidation/src/rule.ts
@@ -1,7 +1,8 @@
 import { RuleResult } from "./rule-result.js";
+import { TsvHost } from "./tsv-host.js";
 
 export interface Rule {
   readonly name: string;
   readonly description: string;
-  execute(folder?: string): Promise<RuleResult>;
+  execute(host?: TsvHost, folder?: string): Promise<RuleResult>;
 }
diff --git a/eng/tools/TypeSpecValidation/src/rules/compile.ts b/eng/tools/TypeSpecValidation/src/rules/compile.ts
@@ -1,30 +1,33 @@
 import path from "path";
-import { runCmd, checkFileExists } from "../utils.js";
 import { Rule } from "../rule.js";
 import { RuleResult } from "../rule-result.js";
+import { TsvHost } from "../tsv-host.js";
 
 export class CompileRule implements Rule {
   readonly name = "Compile";
   readonly description = "Compile TypeSpec";
 
-  async execute(folder: string): Promise<RuleResult> {
+  async execute(host: TsvHost, folder: string): Promise<RuleResult> {
     let success = true;
     let stdOutput = "";
     let errorOutput = "";
 
-    if (await checkFileExists(path.join(folder, "main.tsp"))) {
-      let [err, stdout, stderr] = await runCmd(`npx --no tsp compile . --warn-as-error`, folder);
+    if (await host.checkFileExists(path.join(folder, "main.tsp"))) {
+      let [err, stdout, stderr] = await host.runCmd(
+        `npx --no tsp compile . --warn-as-error`,
+        folder,
+      );
       if (err) {
         success = false;
         errorOutput += err.message;
       }
       stdOutput += stdout;
       errorOutput += stderr;
     }
-    if (await checkFileExists(path.join(folder, "client.tsp"))) {
-      let [err, stdout, stderr] = await runCmd(
+    if (await host.checkFileExists(path.join(folder, "client.tsp"))) {
+      let [err, stdout, stderr] = await host.runCmd(
         `npx --no tsp compile client.tsp --no-emit --warn-as-error`,
-        folder
+        folder,
       );
       if (err) {
         success = false;
diff --git a/eng/tools/TypeSpecValidation/src/rules/folder-structure.ts b/eng/tools/TypeSpecValidation/src/rules/folder-structure.ts
@@ -2,18 +2,18 @@ import { globby } from "globby";
 import path from "path";
 import { Rule } from "../rule.js";
 import { RuleResult } from "../rule-result.js";
-import { checkFileExists } from "../utils.js";
+import { TsvHost } from "../tsv-host.js";
 
 export class FolderStructureRule implements Rule {
   readonly name = "FolderStructure";
   readonly description = "Verify spec directory's folder structure and naming conventions.";
-  async execute(folder: string): Promise<RuleResult> {
+  async execute(host: TsvHost, folder: string): Promise<RuleResult> {
     let success = true;
     let stdOutput = "";
     let errorOutput = "";
 
     stdOutput += `folder: ${folder}\n`;
-    if (!(await checkFileExists(folder))) {
+    if (!(await host.checkFileExists(folder))) {
       return {
         success: false,
         stdOutput: stdOutput,
@@ -64,17 +64,17 @@ export class FolderStructureRule implements Rule {
 
     // Verify tspconfig, main.tsp, examples/
     let containsMinStruct =
-      (await checkFileExists(path.join(folder, "main.tsp"))) ||
-      (await checkFileExists(path.join(folder, "client.tsp")));
+      (await host.checkFileExists(path.join(folder, "main.tsp"))) ||
+      (await host.checkFileExists(path.join(folder, "client.tsp")));
 
-    if (await checkFileExists(path.join(folder, "main.tsp"))) {
+    if (await host.checkFileExists(path.join(folder, "main.tsp"))) {
       containsMinStruct =
-        containsMinStruct && (await checkFileExists(path.join(folder, "examples")));
+        containsMinStruct && (await host.checkFileExists(path.join(folder, "examples")));
     }
 
     if (!packageFolder.includes("Shared")) {
       containsMinStruct =
-        containsMinStruct && (await checkFileExists(path.join(folder, "tspconfig.yaml")));
+        containsMinStruct && (await host.checkFileExists(path.join(folder, "tspconfig.yaml")));
     }
     if (!containsMinStruct) {
       success = false;
diff --git a/eng/tools/TypeSpecValidation/src/rules/format.ts b/eng/tools/TypeSpecValidation/src/rules/format.ts
@@ -1,15 +1,15 @@
 import { Rule } from "../rule.js";
 import { RuleResult } from "../rule-result.js";
-import { runCmd } from "../utils.js";
+import { TsvHost } from "../tsv-host.js";
 
 export class FormatRule implements Rule {
   readonly name = "Format";
   readonly description = "Format TypeSpec";
 
-  async execute(folder: string): Promise<RuleResult> {
+  async execute(host: TsvHost, folder: string): Promise<RuleResult> {
     // Format parent folder to include shared files
 
-    let [err, stdOutput, errorOutput] = await runCmd(`npx tsp format "../**/*.tsp"`, folder);
+    let [err, stdOutput, errorOutput] = await host.runCmd(`npx tsp format "../**/*.tsp"`, folder);
     // Failing on both err and errorOutput because of known bug in tsp format where it returns 0 on failed formatting
     // https://github.com/microsoft/typespec/issues/2323
     let success = !err && !errorOutput;
diff --git a/eng/tools/TypeSpecValidation/src/rules/git-diff.ts b/eng/tools/TypeSpecValidation/src/rules/git-diff.ts
@@ -1,13 +1,13 @@
-import { simpleGit } from "simple-git";
 import { Rule } from "../rule.js";
 import { RuleResult } from "../rule-result.js";
+import { TsvHost } from "../tsv-host.js";
 
 export class GitDiffRule implements Rule {
   readonly name = "GitDiff";
   readonly description = "Checks if previous rules resulted in a git diff";
 
-  async execute(folder: string): Promise<RuleResult> {
-    const git = simpleGit(folder);
+  async execute(host: TsvHost, folder: string): Promise<RuleResult> {
+    const git = host.gitOperation(folder);
     let gitStatusIsClean = (await git.status(["--porcelain"])).isClean();
 
     let success = true;
diff --git a/eng/tools/TypeSpecValidation/src/rules/linter-ruleset.ts b/eng/tools/TypeSpecValidation/src/rules/linter-ruleset.ts
@@ -3,15 +3,15 @@ import { join } from "path";
 import { parse as yamlParse } from "yaml";
 import { Rule } from "../rule.js";
 import { RuleResult } from "../rule-result.js";
-import { checkFileExists } from "../utils.js";
+import { TsvHost } from "../tsv-host.js";
 
 export class LinterRulesetRule implements Rule {
   readonly name = "LinterRuleset";
 
   readonly description =
     "Ensures each spec includes the correct linter ruleset (data-plane or management-plane)";
 
-  async execute(folder: string): Promise<RuleResult> {
+  async execute(host: TsvHost, folder: string): Promise<RuleResult> {
     let success = true;
     let stdOutput = "";
     let errorOutput = "";
@@ -24,8 +24,8 @@ export class LinterRulesetRule implements Rule {
       config.options?.["@azure-tools/typespec-autorest"]?.["azure-resource-provider-folder"];
     stdOutput += `azure-resource-provider-folder: ${JSON.stringify(rpFolder)}\n`;
 
-    const mainTspExists = await checkFileExists(join(folder, "main.tsp"));
-    const clientTspExists = await checkFileExists(join(folder, "client.tsp"));
+    const mainTspExists = await host.checkFileExists(join(folder, "main.tsp"));
+    const clientTspExists = await host.checkFileExists(join(folder, "client.tsp"));
     let files = [];
     if (mainTspExists) {
       files.push("main.tsp");
diff --git a/eng/tools/TypeSpecValidation/src/rules/npm-prefix.ts b/eng/tools/TypeSpecValidation/src/rules/npm-prefix.ts
@@ -1,15 +1,14 @@
 import path from "path";
-import { simpleGit } from "simple-git";
-import { runCmd } from "../utils.js";
 import { Rule } from "../rule.js";
 import { RuleResult } from "../rule-result.js";
+import { TsvHost } from "../tsv-host.js";
 
 export class NpmPrefixRule implements Rule {
   readonly name = "NpmPrefix";
   readonly description = "Verify spec is using root level package.json";
 
-  async execute(folder: string): Promise<RuleResult> {
-    const git = simpleGit(folder);
+  async execute(host: TsvHost, folder: string): Promise<RuleResult> {
+    const git = host.gitOperation(folder);
 
     let expected_npm_prefix: string | undefined;
     try {
@@ -23,7 +22,7 @@ export class NpmPrefixRule implements Rule {
       };
     }
 
-    const actual_npm_prefix = path.normalize((await runCmd(`npm prefix`, folder))[1].trim());
+    const actual_npm_prefix = path.normalize((await host.runCmd(`npm prefix`, folder))[1].trim());
 
     let success = true;
     let stdOutput =
diff --git a/eng/tools/TypeSpecValidation/src/tsv-host.ts b/eng/tools/TypeSpecValidation/src/tsv-host.ts
@@ -0,0 +1,12 @@
+export interface TsvHost {
+  gitOperation(folder: string): IGitOperation;
+  runCmd(cmd: string, cwd: string): Promise<[Error | null, string, string]>;
+  checkFileExists(file: string): Promise<boolean>;
+  // TODO: Other functions that need mocks
+}
+
+export interface IGitOperation {
+  status(options?: string[]): Promise<{ isClean(): boolean; modified: string[] }>;
+  diff(): Promise<string>;
+  revparse(option: string): Promise<string>;
+}
diff --git a/eng/tools/TypeSpecValidation/src/tsv-runner-host.ts b/eng/tools/TypeSpecValidation/src/tsv-runner-host.ts
@@ -0,0 +1,10 @@
+import { TsvHost } from "./tsv-host.js";
+import { simpleGit } from "simple-git";
+import { runCmd, checkFileExists } from "./utils.js";
+
+export const TsvRunnerHost: TsvHost = {
+  gitOperation: simpleGit,
+  runCmd: runCmd,
+  checkFileExists: checkFileExists,
+  // TODO: Other functions that need mocks
+};
diff --git a/eng/tools/TypeSpecValidation/src/utils.ts b/eng/tools/TypeSpecValidation/src/utils.ts
@@ -8,11 +8,11 @@ export async function runCmd(cmd: string, cwd: string) {
       cmd,
       { encoding: "utf8", maxBuffer: 1024 * 1024 * 64, cwd: cwd },
       (err: unknown, stdout: unknown, stderr: unknown) =>
-        res({ err: err, stdout: stdout, stderr: stderr })
-    )
+        res({ err: err, stdout: stdout, stderr: stderr }),
+    ),
   )) as any;
 
-  return [err, stdout, stderr];
+  return [err, stdout, stderr] as [Error | null, string, string];
 }
 
 export async function checkFileExists(file: string) {
diff --git a/eng/tools/TypeSpecValidation/test/compile.test.ts b/eng/tools/TypeSpecValidation/test/compile.test.ts
@@ -0,0 +1,15 @@
+// import { error } from "node:console";
+import { CompileRule } from "../src/rules/compile.js";
+import { TsvTestHostGenerator } from "./tsv-test-host.js";
+import { strict as assert } from "node:assert";
+describe("#Compile", function () {
+  it("Should fail if project cannot compile.", async function () {
+    let compileRule = new CompileRule();
+    let TsvTestHost = TsvTestHostGenerator({
+      runCmd: Promise.resolve([null, "success", "success"]),
+    });
+    const result = await compileRule.execute(TsvTestHost, "mockFolder");
+    assert(result.success);
+    assert(result.stdOutput === "successsuccess");
+  });
+});
diff --git a/eng/tools/TypeSpecValidation/test/folder-structure.test.ts b/eng/tools/TypeSpecValidation/test/folder-structure.test.ts
@@ -0,0 +1,5 @@
+describe("#FolderStructure", function () {
+  it("Should fail if directory name does not conform.", async function () {
+    // TODO: Folder structure rule unit test
+  });
+});
diff --git a/eng/tools/TypeSpecValidation/test/git-diff.test.ts b/eng/tools/TypeSpecValidation/test/git-diff.test.ts
@@ -0,0 +1,12 @@
+import { GitDiffRule } from "../src/rules/git-diff.js";
+import { DefaultTsvTestHost } from "./tsv-test-host.js";
+import { strict as assert } from "node:assert";
+
+describe("#GitDiff", function () {
+  it("Should fail if git diff produces output.", async function () {
+    let gitDiffRule = new GitDiffRule();
+    const result = await gitDiffRule.execute(DefaultTsvTestHost, "mockFolder");
+    assert(result.errorOutput);
+    assert(result.errorOutput.length !== 0);
+  });
+});
diff --git a/eng/tools/TypeSpecValidation/test/tsv-test-host.ts b/eng/tools/TypeSpecValidation/test/tsv-test-host.ts
@@ -0,0 +1,57 @@
+import { TsvHost } from "../src/tsv-host.js";
+
+export function TsvTestHostGenerator(fnReturnSub: { [key: string]: Object }) {
+  let tsvTestHost = Object.assign({}, DefaultTsvTestHost); // Needs deep copy
+  for (let key in fnReturnSub) {
+    let typedKey = key as keyof TsvHost;
+    if (fnReturnSub[typedKey]) {
+      tsvTestHost = {
+        ...tsvTestHost,
+        [typedKey]: () => {
+          return fnReturnSub[typedKey];
+        },
+      };
+    } else {
+      tsvTestHost = {
+        ...tsvTestHost,
+        [typedKey]: () => {},
+      };
+    }
+  }
+  return tsvTestHost;
+}
+
+export const DefaultTsvTestHost: TsvHost = {
+  gitOperation: (folder: string) => {
+    return {
+      status: () => {
+        return Promise.resolve({
+          modified: [`${folder}\n`],
+          isClean: () => {
+            return false;
+          },
+        });
+      },
+      diff: () => {
+        return Promise.resolve("diff");
+      },
+      revparse: () => {
+        return Promise.resolve("");
+      },
+    };
+  },
+
+  runCmd: async (cmd: string, cwd: string) => {
+    let err = null;
+    let stdout = `default ${cmd} at ${cwd}`;
+    let stderr = "";
+
+    return [err, stdout, stderr];
+  },
+
+  checkFileExists: async (file: string) => {
+    console.log(`file ${file} exists`);
+    return true;
+  },
+  // TODO: Other functions that need mocks
+};
diff --git a/package-lock.json b/package-lock.json

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,8 @@`
`1`	`1`	`import { RuleResult } from "./rule-result.js";`
	`2`	`+import { TsvHost } from "./tsv-host.js";`
`2`	`3`
`3`	`4`	`export interface Rule {`
`4`	`5`	`readonly name: string;`
`5`	`6`	`readonly description: string;`
`6`		`- execute(folder?: string): Promise<RuleResult>;`
	`7`	`+ execute(host?: TsvHost, folder?: string): Promise<RuleResult>;`
`7`	`8`	`}`