From f00a51213e14a2c45838969bfcdc156e51461714 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Sun, 14 Sep 2025 18:09:22 +0200 Subject: [PATCH 1/2] refactor(vitest-plugin): make the walltimerunner class less cluttered --- packages/vitest-plugin/rollup.config.ts | 2 +- packages/vitest-plugin/src/walltime.ts | 186 ------------------- packages/vitest-plugin/src/walltime/index.ts | 37 ++++ packages/vitest-plugin/src/walltime/utils.ts | 130 +++++++++++++ 4 files changed, 168 insertions(+), 187 deletions(-) delete mode 100644 packages/vitest-plugin/src/walltime.ts create mode 100644 packages/vitest-plugin/src/walltime/index.ts create mode 100644 packages/vitest-plugin/src/walltime/utils.ts diff --git a/packages/vitest-plugin/rollup.config.ts b/packages/vitest-plugin/rollup.config.ts index 03e611ec..5fbbc655 100644 --- a/packages/vitest-plugin/rollup.config.ts +++ b/packages/vitest-plugin/rollup.config.ts @@ -27,7 +27,7 @@ export default defineConfig([ external: ["@codspeed/core", /^vitest/], }, { - input: "src/walltime.ts", + input: "src/walltime/index.ts", output: { file: "dist/walltime.mjs", format: "es" }, plugins: jsPlugins(pkg.version), external: ["@codspeed/core", /^vitest/], diff --git a/packages/vitest-plugin/src/walltime.ts b/packages/vitest-plugin/src/walltime.ts deleted file mode 100644 index 9eda348b..00000000 --- a/packages/vitest-plugin/src/walltime.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { - calculateQuantiles, - msToNs, - msToS, - writeWalltimeResults, - type Benchmark, - type BenchmarkStats, -} from "@codspeed/core"; -import { - type Benchmark as VitestBenchmark, - type RunnerTaskResult, - type RunnerTestSuite, -} from "vitest"; -import { NodeBenchmarkRunner } from "vitest/runners"; -import { getBenchOptions } from "vitest/suite"; -import { - isVitestTaskBenchmark, - patchRootSuiteWithFullFilePath, -} from "./common"; - -declare const __VERSION__: string; - -/** - * WalltimeRunner uses Vitest's default benchmark execution - * and extracts results from the suite after completion - */ -export class WalltimeRunner extends NodeBenchmarkRunner { - async runSuite(suite: RunnerTestSuite): Promise { - patchRootSuiteWithFullFilePath(suite); - - console.log( - `[CodSpeed] running with @codspeed/vitest-plugin v${__VERSION__} (walltime mode)` - ); - - // Let Vitest's default benchmark runner handle execution - await super.runSuite(suite); - - // Extract benchmark results from the completed suite - const benchmarks = await this.extractBenchmarkResults(suite); - - if (benchmarks.length > 0) { - writeWalltimeResults(benchmarks); - console.log( - `[CodSpeed] Done collecting walltime data for ${benchmarks.length} benches.` - ); - } else { - console.warn( - `[CodSpeed] No benchmark results found after suite execution` - ); - } - } - - private async extractBenchmarkResults( - suite: RunnerTestSuite, - parentPath = "" - ): Promise { - const benchmarks: Benchmark[] = []; - const currentPath = parentPath - ? `${parentPath}::${suite.name}` - : suite.name; - - for (const task of suite.tasks) { - if (isVitestTaskBenchmark(task) && task.result?.state === "pass") { - const benchmark = await this.processBenchmarkTask(task, currentPath); - if (benchmark) { - benchmarks.push(benchmark); - } - } else if (task.type === "suite") { - const nestedBenchmarks = await this.extractBenchmarkResults( - task, - currentPath - ); - benchmarks.push(...nestedBenchmarks); - } - } - - return benchmarks; - } - - private async processBenchmarkTask( - task: VitestBenchmark, - suitePath: string - ): Promise { - const uri = `${suitePath}::${task.name}`; - - const result = task.result; - if (!result) { - console.warn(` ⚠ No result data available for ${uri}`); - return null; - } - - try { - // Get tinybench configuration options from vitest - const benchOptions = getBenchOptions(task); - - const stats = this.convertVitestResultToBenchmarkStats( - result, - benchOptions - ); - - if (stats === null) { - console.log(` ✔ No walltime data to collect for ${uri}`); - return null; - } - - const coreBenchmark: Benchmark = { - name: task.name, - uri, - config: { - max_rounds: benchOptions.iterations ?? null, - max_time_ns: benchOptions.time ? msToNs(benchOptions.time) : null, - min_round_time_ns: null, // tinybench does not have an option for this - warmup_time_ns: - benchOptions.warmupIterations !== 0 && benchOptions.warmupTime - ? msToNs(benchOptions.warmupTime) - : null, - }, - stats, - }; - - console.log(` ✔ Collected walltime data for ${uri}`); - return coreBenchmark; - } catch (error) { - console.warn( - ` ⚠ Failed to process benchmark result for ${uri}:`, - error - ); - return null; - } - } - - private convertVitestResultToBenchmarkStats( - result: RunnerTaskResult, - benchOptions: { - time?: number; - warmupTime?: number; - warmupIterations?: number; - iterations?: number; - } - ): BenchmarkStats | null { - const benchmark = result.benchmark; - - if (!benchmark) { - throw new Error("No benchmark data available in result"); - } - - const { totalTime, min, max, mean, sd, samples } = benchmark; - - // Get individual sample times in nanoseconds and sort them - const sortedTimesNs = samples.map(msToNs).sort((a, b) => a - b); - const meanNs = msToNs(mean); - const stdevNs = msToNs(sd); - - if (sortedTimesNs.length == 0) { - // Sometimes the benchmarks can be completely optimized out and not even run, but its beforeEach and afterEach hooks are still executed, and the task is still considered a success. - // This is the case for the hooks.bench.ts example in this package - return null; - } - - const { - q1_ns, - q3_ns, - median_ns, - iqr_outlier_rounds, - stdev_outlier_rounds, - } = calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); - - return { - min_ns: msToNs(min), - max_ns: msToNs(max), - mean_ns: meanNs, - stdev_ns: stdevNs, - q1_ns, - median_ns, - q3_ns, - total_time: msToS(totalTime), - iter_per_round: 1, // as there is only one round in tinybench, we define that there were n rounds of 1 iteration - rounds: sortedTimesNs.length, - iqr_outlier_rounds, - stdev_outlier_rounds, - warmup_iters: benchOptions.warmupIterations ?? 0, - }; - } -} - -export default WalltimeRunner; diff --git a/packages/vitest-plugin/src/walltime/index.ts b/packages/vitest-plugin/src/walltime/index.ts new file mode 100644 index 00000000..96f8a923 --- /dev/null +++ b/packages/vitest-plugin/src/walltime/index.ts @@ -0,0 +1,37 @@ +import { setupCore, writeWalltimeResults } from "@codspeed/core"; +import { type RunnerTestSuite } from "vitest"; +import { NodeBenchmarkRunner } from "vitest/runners"; +import { patchRootSuiteWithFullFilePath } from "../common"; +import { extractBenchmarkResults } from "./utils"; + +/** + * WalltimeRunner uses Vitest's default benchmark execution + * and extracts results from the suite after completion + */ +export class WalltimeRunner extends NodeBenchmarkRunner { + private isTinybenchHookedWithCodspeed = false; + private benchmarkUris = new Map(); + + async runSuite(suite: RunnerTestSuite): Promise { + patchRootSuiteWithFullFilePath(suite); + + setupCore(); + + await super.runSuite(suite); + + const benchmarks = await extractBenchmarkResults(suite); + + if (benchmarks.length > 0) { + writeWalltimeResults(benchmarks); + console.log( + `[CodSpeed] Done collecting walltime data for ${benchmarks.length} benches.` + ); + } else { + console.warn( + `[CodSpeed] No benchmark results found after suite execution` + ); + } + } +} + +export default WalltimeRunner; diff --git a/packages/vitest-plugin/src/walltime/utils.ts b/packages/vitest-plugin/src/walltime/utils.ts new file mode 100644 index 00000000..db31d962 --- /dev/null +++ b/packages/vitest-plugin/src/walltime/utils.ts @@ -0,0 +1,130 @@ +import { + calculateQuantiles, + msToNs, + msToS, + type Benchmark, + type BenchmarkStats, +} from "@codspeed/core"; +import { + type Benchmark as VitestBenchmark, + type RunnerTaskResult, + type RunnerTestSuite, +} from "vitest"; +import { getBenchOptions } from "vitest/suite"; +import { isVitestTaskBenchmark } from "../common"; + +export async function extractBenchmarkResults( + suite: RunnerTestSuite, + parentPath = "" +): Promise { + const benchmarks: Benchmark[] = []; + const currentPath = parentPath ? `${parentPath}::${suite.name}` : suite.name; + + for (const task of suite.tasks) { + if (isVitestTaskBenchmark(task) && task.result?.state === "pass") { + const benchmark = await processBenchmarkTask(task, currentPath); + if (benchmark) { + benchmarks.push(benchmark); + } + } else if (task.type === "suite") { + const nestedBenchmarks = await extractBenchmarkResults(task, currentPath); + benchmarks.push(...nestedBenchmarks); + } + } + + return benchmarks; +} + +async function processBenchmarkTask( + task: VitestBenchmark, + suitePath: string +): Promise { + const uri = `${suitePath}::${task.name}`; + + const result = task.result; + if (!result) { + console.warn(` ⚠ No result data available for ${uri}`); + return null; + } + + try { + // Get tinybench configuration options from vitest + const benchOptions = getBenchOptions(task); + + const stats = convertVitestResultToBenchmarkStats(result, benchOptions); + + if (stats === null) { + console.log(` ✔ No walltime data to collect for ${uri}`); + return null; + } + + const coreBenchmark: Benchmark = { + name: task.name, + uri, + config: { + max_rounds: benchOptions.iterations ?? null, + max_time_ns: benchOptions.time ? msToNs(benchOptions.time) : null, + min_round_time_ns: null, // tinybench does not have an option for this + warmup_time_ns: + benchOptions.warmupIterations !== 0 && benchOptions.warmupTime + ? msToNs(benchOptions.warmupTime) + : null, + }, + stats, + }; + + console.log(` ✔ Collected walltime data for ${uri}`); + return coreBenchmark; + } catch (error) { + console.warn(` ⚠ Failed to process benchmark result for ${uri}:`, error); + return null; + } +} + +function convertVitestResultToBenchmarkStats( + result: RunnerTaskResult, + benchOptions: { + time?: number; + warmupTime?: number; + warmupIterations?: number; + iterations?: number; + } +): BenchmarkStats | null { + const benchmark = result.benchmark; + + if (!benchmark) { + throw new Error("No benchmark data available in result"); + } + + const { totalTime, min, max, mean, sd, samples } = benchmark; + + // Get individual sample times in nanoseconds and sort them + const sortedTimesNs = samples.map(msToNs).sort((a, b) => a - b); + const meanNs = msToNs(mean); + const stdevNs = msToNs(sd); + + if (sortedTimesNs.length == 0) { + // Sometimes the benchmarks can be completely optimized out and not even run, but its beforeEach and afterEach hooks are still executed, and the task is still considered a success. + // This is the case for the hooks.bench.ts example in this package + return null; + } + + const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } = + calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); + + return { + min_ns: msToNs(min), + max_ns: msToNs(max), + mean_ns: meanNs, + stdev_ns: stdevNs, + q1_ns, + median_ns, + q3_ns, + total_time: msToS(totalTime), + iter_per_round: 1, // as there is only one round in tinybench, we define that there were n rounds of 1 iteration + rounds: sortedTimesNs.length, + iqr_outlier_rounds, + stdev_outlier_rounds, + warmup_iters: benchOptions.warmupIterations ?? 0, + }; +} From 453da47cb2c90340f0aa95443b510721ba1235a9 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Sun, 14 Sep 2025 18:10:03 +0200 Subject: [PATCH 2/2] feat(vitest-plugin): add perf profiling for vitest plugin --- packages/vitest-plugin/package.json | 2 + packages/vitest-plugin/src/walltime/index.ts | 84 +++++++++++++++++++- pnpm-lock.yaml | 3 + 3 files changed, 86 insertions(+), 3 deletions(-) diff --git a/packages/vitest-plugin/package.json b/packages/vitest-plugin/package.json index 3a4c336c..d3535bf2 100644 --- a/packages/vitest-plugin/package.json +++ b/packages/vitest-plugin/package.json @@ -31,12 +31,14 @@ "@codspeed/core": "workspace:^5.0.0" }, "peerDependencies": { + "tinybench": "^2.9.0", "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0", "vitest": ">=3.2" }, "devDependencies": { "@total-typescript/shoehorn": "^0.1.1", "execa": "^8.0.1", + "tinybench": "^2.9.0", "vite": "^7.0.0", "vitest": "^3.2.4" } diff --git a/packages/vitest-plugin/src/walltime/index.ts b/packages/vitest-plugin/src/walltime/index.ts index 96f8a923..fa4d8524 100644 --- a/packages/vitest-plugin/src/walltime/index.ts +++ b/packages/vitest-plugin/src/walltime/index.ts @@ -1,5 +1,14 @@ -import { setupCore, writeWalltimeResults } from "@codspeed/core"; -import { type RunnerTestSuite } from "vitest"; +import { + InstrumentHooks, + setupCore, + writeWalltimeResults, +} from "@codspeed/core"; +import { Fn } from "tinybench"; +import { + RunnerTaskEventPack, + RunnerTaskResultPack, + type RunnerTestSuite, +} from "vitest"; import { NodeBenchmarkRunner } from "vitest/runners"; import { patchRootSuiteWithFullFilePath } from "../common"; import { extractBenchmarkResults } from "./utils"; @@ -10,10 +19,13 @@ import { extractBenchmarkResults } from "./utils"; */ export class WalltimeRunner extends NodeBenchmarkRunner { private isTinybenchHookedWithCodspeed = false; - private benchmarkUris = new Map(); + private suiteUris = new Map(); + /// Suite ID of the currently running suite, to allow constructing the URI in the context of tinybench tasks + private currentSuiteId: string | null = null; async runSuite(suite: RunnerTestSuite): Promise { patchRootSuiteWithFullFilePath(suite); + this.populateBenchmarkUris(suite); setupCore(); @@ -32,6 +44,72 @@ export class WalltimeRunner extends NodeBenchmarkRunner { ); } } + + private populateBenchmarkUris(suite: RunnerTestSuite, parentPath = ""): void { + const currentPath = + parentPath !== "" ? `${parentPath}::${suite.name}` : suite.name; + + for (const task of suite.tasks) { + if (task.type === "suite") { + this.suiteUris.set(task.id, `${currentPath}::${task.name}`); + this.populateBenchmarkUris(task, currentPath); + } + } + } + + async importTinybench(): Promise { + const tinybench = await super.importTinybench(); + + if (this.isTinybenchHookedWithCodspeed) { + return tinybench; + } + this.isTinybenchHookedWithCodspeed = true; + + const originalRun = tinybench.Task.prototype.run; + + const getSuiteUri = (): string => { + if (this.currentSuiteId === null) { + throw new Error("currentSuiteId is null - something went wrong"); + } + return this.suiteUris.get(this.currentSuiteId) || ""; + }; + + tinybench.Task.prototype.run = async function () { + const { fn } = this as { fn: Fn }; + const suiteUri = getSuiteUri(); + + function __codspeed_root_frame__() { + return fn(); + } + (this as { fn: Fn }).fn = __codspeed_root_frame__; + + InstrumentHooks.startBenchmark(); + await originalRun.call(this); + InstrumentHooks.stopBenchmark(); + + // Look up the URI by task name + const uri = `${suiteUri}::${this.name}`; + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + + return this; + }; + + return tinybench; + } + + // Allow tinybench to retrieve the path to the currently running suite + async onTaskUpdate( + _: RunnerTaskResultPack[], + events: RunnerTaskEventPack[] + ): Promise { + events.map((event) => { + const [id, eventName] = event; + + if (eventName === "suite-prepare") { + this.currentSuiteId = id; + } + }); + } } export default WalltimeRunner; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 98e6f532..4bbeeab1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -306,6 +306,9 @@ importers: execa: specifier: ^8.0.1 version: 8.0.1 + tinybench: + specifier: ^2.9.0 + version: 2.9.0 vite: specifier: ^7.0.0 version: 7.1.3(@types/node@20.19.11)