implement example apps benchmarking

Author: dario-piotrowicz

benchmarking/.gitignore

@@ -0,0 +1 @@
+results/

benchmarking/README.md

@@ -0,0 +1,7 @@
+# Benchmarking
+
+This directory contains a script for running full end to end benchmarks again the example applications
+
+> [!note]
+> This is the first cut at benchmarking our solution, later we can take the script in this directory,
+> generalize it and make it more reusable if we want

benchmarking/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "@opennextjs-cloudflare/benchmarking",
+  "private": true,
+  "type": "module",
+  "devDependencies": {
+    "tsx": "catalog:",
+    "@tsconfig/strictest": "catalog:",
+    "@types/node": "catalog:",
+    "ora": "^8.1.0"
+  },
+  "scripts": {
+    "benchmark": "tsx src/index.ts"
+  }
+}

benchmarking/src/benchmarking.ts

@@ -0,0 +1,110 @@
+import nodeTimesPromises from "node:timers/promises";
+import nodeFsPromises from "node:fs/promises";
+import nodePath from "node:path";
+
+export type FetchBenchmark = {
+  calls: number[];
+  average: number;
+};
+
+export type BenchmarkingResults = {
+  name: string;
+  path: string;
+  fetchBenchmark: FetchBenchmark;
+}[];
+
+/**
+ * Benchmarks the response time of an application end-to-end by:
+ *  - building the application
+ *  - deploying it
+ *  - and fetching from it (multiple times)
+ *
+ * @param options.build function implementing how the application is to be built
+ * @param options.deploy function implementing how the application is deployed (returning the url of the deployment)
+ * @param options.fetch function indicating how to fetch from the application (in case a specific route needs to be hit, cookies need to be applied, etc...)
+ * @returns the benchmarking results for the application
+ */
+export async function benchmarkApplicationResponseTime({
+  build,
+  deploy,
+  fetch,
+}: {
+  build: () => Promise<void>;
+  deploy: () => Promise<string>;
+  fetch: (deploymentUrl: string) => Promise<Response>;
+}): Promise<FetchBenchmark> {
+  await build();
+  const deploymentUrl = await deploy();
+  return benchmarkFetch(deploymentUrl, { fetch });
+}
+
+type BenchmarkFetchOptions = {
+  numberOfCalls?: number;
+  randomDelayMax?: number;
+  fetch: (deploymentUrl: string) => Promise<Response>;
+};
+
+const defaultOptions: Required<Omit<BenchmarkFetchOptions, "fetch">> = {
+  numberOfCalls: 20,
+  randomDelayMax: 15_000,
+};
+
+/**
+ * Benchmarks a fetch operation by running it multiple times and computing the average time (in milliseconds) such fetch operation takes.
+ *
+ * @param url The url to fetch from
+ * @param options options for the benchmarking
+ * @returns the computed average alongside all the single call times
+ */
+async function benchmarkFetch(url: string, options: BenchmarkFetchOptions): Promise<FetchBenchmark> {
+  const benchmarkFetchCall = async () => {
+    const preTime = performance.now();
+    const resp = await options.fetch(url);
+    const postTime = performance.now();
+
+    if (!resp.ok) {
+      throw new Error(`Error: Failed to fetch from "${url}"`);
+    }
+
+    return postTime - preTime;
+  };
+
+  const calls = await Promise.all(
+    new Array(options?.numberOfCalls ?? defaultOptions.numberOfCalls).fill(null).map(async () => {
+      // let's add a random delay before we make the fetch
+      await nodeTimesPromises.setTimeout(
+        Math.round(Math.random() * (options?.randomDelayMax ?? defaultOptions.randomDelayMax))
+      );
+
+      return benchmarkFetchCall();
+    })
+  );
+
+  const average = calls.reduce((time, sum) => sum + time) / calls.length;
+
+  return {
+    calls,
+    average,
+  };
+}
+
+/**
+ * Saves benchmarking results in a local json file
+ *
+ * @param results the benchmarking results to save
+ * @returns the path to the created json file
+ */
+export async function saveResultsToDisk(results: BenchmarkingResults): Promise<string> {
+  const date = new Date();
+
+  const fileName = `${date.toISOString().split(".")[0]!.replace("T", "_").replaceAll(":", "-")}.json`;
+
+  const outputFile = nodePath.resolve(`./results/${fileName}`);
+
+  await nodeFsPromises.mkdir(nodePath.dirname(outputFile), { recursive: true });
+
+  const resultStr = JSON.stringify(results, null, 2);
+  await nodeFsPromises.writeFile(outputFile, resultStr);
+
+  return outputFile;
+}

benchmarking/src/cloudflare.ts

@@ -0,0 +1,95 @@
+import nodeFsPromises from "node:fs/promises";
+import nodeFs from "node:fs";
+import nodePath from "node:path";
+import nodeChildProcess from "node:child_process";
+import nodeUtil from "node:util";
+
+const promiseExec = nodeUtil.promisify(nodeChildProcess.exec);
+
+await ensureWranglerSetup();
+
+/**
+ * Collects name and absolute paths of apps (in this repository) that we want to benchmark
+ *
+ * @returns Array of objects containing the app's name and absolute path
+ */
+export async function collectAppPathsToBenchmark(): Promise<
+  {
+    name: string;
+    path: string;
+  }[]
+> {
+  const allExampleNames = await nodeFsPromises.readdir("../examples");
+
+  const examplesToIgnore = new Set(["vercel-commerce"]);
+
+  const examplePaths = allExampleNames
+    .filter((exampleName) => !examplesToIgnore.has(exampleName))
+    .map((exampleName) => ({
+      name: exampleName,
+      path: nodePath.resolve(`../examples/${exampleName}`),
+    }));
+
+  return examplePaths;
+}
+
+/**
+ * Builds an application using their "build:worker" script
+ * (an error is thrown if the application doesn't have such a script)
+ *
+ * @param dir Path to the application to build
+ */
+export async function buildApp(dir: string): Promise<void> {
+  const packageJsonPath = `${dir}/package.json`;
+  if (!nodeFs.existsSync(packageJsonPath)) {
+    throw new Error(`Error: package.json for app at "${dir}" not found`);
+  }
+
+  const packageJsonContent = JSON.parse(await nodeFsPromises.readFile(packageJsonPath, "utf8"));
+
+  if (!("scripts" in packageJsonContent) || !("build:worker" in packageJsonContent.scripts)) {
+    throw new Error(`Error: package.json for app at "${dir}" does not include a "build:worker" script`);
+  }
+
+  const command = "pnpm build:worker";
+
+  await promiseExec(command, { cwd: dir });
+}
+
+/**
+ * Deploys a built application using wrangler
+ *
+ * @param dir Path to the application to build
+ * @returns the url of the deployed application
+ */
+export async function deployBuiltApp(dir: string): Promise<string> {
+  const { stdout } = await promiseExec("pnpm exec wrangler deploy", { cwd: dir });
+
+  const deploymentUrl = stdout.match(/\bhttps:\/\/(?:[a-zA-Z0-9.\-])*\.workers\.dev\b/)?.[0];
+
+  if (!deploymentUrl) {
+    throw new Error(`Could not obtain a deployment url for app at "${dir}"`);
+  }
+
+  return deploymentUrl;
+}
+
+/**
+ * Makes sure that everything is set up so that wrangler can actually deploy the applications.
+ * This means that:
+ *  - the user has logged in
+ *  - if they have more than one account they have set a CLOUDFLARE_ACCOUNT_ID env variable
+ */
+async function ensureWranglerSetup(): Promise<void> {
+  const { stdout } = await promiseExec("pnpm dlx wrangler whoami");
+
+  if (stdout.includes("You are not authenticated")) {
+    throw new Error("Please log in using wrangler by running `pnpm dlx wrangler login`");
+  }
+
+  if (!(process.env as Record<string, unknown>)["CLOUDFLARE_ACCOUNT_ID"]) {
+    throw new Error(
+      "Please set the CLOUDFLARE_ACCOUNT_ID environment variable to the id of the account you want to use to deploy the applications"
+    );
+  }
+}

benchmarking/src/index.ts

@@ -0,0 +1,41 @@
+import nodeTimesPromises from "node:timers/promises";
+import * as cloudflare from "./cloudflare";
+import { benchmarkApplicationResponseTime, BenchmarkingResults, saveResultsToDisk } from "./benchmarking";
+import { runOperationsWithSpinner } from "./utils";
+
+const appPathsToBenchmark = await cloudflare.collectAppPathsToBenchmark();
+
+const benchmarkingResults: BenchmarkingResults = await runOperationsWithSpinner(
+  "Benchmarking Apps",
+  appPathsToBenchmark.map(({ name, path }, i) => async () => {
+    await nodeTimesPromises.setTimeout(i * 1_000);
+    const fetchBenchmark = await benchmarkApplicationResponseTime({
+      build: async () => cloudflare.buildApp(path),
+      deploy: async () => cloudflare.deployBuiltApp(path),
+      fetch,
+    });
+
+    return {
+      name,
+      path,
+      fetchBenchmark,
+    };
+  })
+);
+
+console.log();
+
+const outputFile = await saveResultsToDisk(benchmarkingResults);
+
+console.log(`The benchmarking results have been written in ${outputFile}`);
+
+console.log("\n\nSummary: ");
+const summary = benchmarkingResults.map(({ name, fetchBenchmark }) => ({
+  name,
+  "average fetch duration (ms)": Math.round(fetchBenchmark.average),
+}));
+console.table(summary);
+
+console.log();
+
+process.exit(0);

benchmarking/src/utils.ts

@@ -0,0 +1,39 @@
+import ora from "ora";
+
+/**
+ * Runs a number of operations while presenting a loading spinner with some text
+ *
+ * @param spinnerText The text to add to the spinner
+ * @param operations The operations to run
+ * @returns The operations results
+ */
+export async function runOperationsWithSpinner<T>(
+  spinnerText: string,
+  operations: (() => Promise<T>)[]
+): Promise<T[]> {
+  const spinner = ora({
+    discardStdin: false,
+    hideCursor: false,
+  }).start();
+
+  let doneCount = 0;
+
+  const updateSpinnerText = () => {
+    doneCount++;
+    spinner.text = `${spinnerText} (${doneCount}/${operations.length})`;
+  };
+
+  updateSpinnerText();
+
+  const results = await Promise.all(
+    operations.map(async (operation) => {
+      const result = await operation();
+      updateSpinnerText();
+      return result;
+    })
+  );
+
+  spinner.stop();
+
+  return results;
+}

benchmarking/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "$schema": "https://json.schemastore.org/tsconfig",
+  "extends": "@tsconfig/strictest/tsconfig.json",
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "ESNext",
+    "lib": ["ESNext"],
+    "types": ["node"],
+    "moduleResolution": "Bundler",
+    "forceConsistentCasingInFileNames": true,
+    "noImplicitReturns": false,
+    "exactOptionalPropertyTypes": false
+  },
+  "include": ["./src/**/*.ts"]
+}

package.json

@@ -21,6 +21,7 @@
     "postinstall": "pnpm --filter cloudflare build",
     "build": "pnpm --filter cloudflare build",
     "e2e": "pnpm build && pnpm -r e2e",
-    "e2e:dev": "pnpm build && pnpm -r e2e:dev"
+    "e2e:dev": "pnpm build && pnpm -r e2e:dev",
+    "benchmark": "pnpm run --filter benchmarking benchmark"
   }
 }