WC-4185 Support 100k assets when jwt claim more_files is present

As noted in the comments, normally we'd need to validate this jwt, but
since any uploads depend on a valid jwt (and are validated later) it's
fine to just decode the jwt and check for this feature

Author: matthewdavidrodgers

.changeset/two-comics-win.md

@@ -0,0 +1,5 @@
+---
+"wrangler": minor
+---
+
+Support more files in an upload for non-free users

packages/wrangler/src/__tests__/pages/project-upload.test.ts

@@ -2,6 +2,7 @@
 import { mkdirSync, writeFileSync } from "node:fs";
 import { http, HttpResponse } from "msw";
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { isMoreFilesEnabled } from "../../pages/upload";
 import { endEventLoop } from "../helpers/end-event-loop";
 import { mockAccountId, mockApiToken } from "../helpers/mock-account-id";
 import { mockConsoleMethods } from "../helpers/mock-console";
@@ -665,3 +666,44 @@ describe("pages project upload", () => {
 		expect(std.err).toMatchInlineSnapshot(`""`);
 	});
 });
+
+describe("isMoreFilesEnabled", () => {
+	it("should return true when JWT has more_files feature in claims", () => {
+		// JWT payload: {"more_files": true}
+		const jwt =
+			"header." +
+			Buffer.from(
+				JSON.stringify({ features: ["files", "more_files"] })
+			).toString("base64") +
+			".signature";
+		expect(isMoreFilesEnabled(jwt)).toBe(true);
+	});
+
+	it("should return false when JWT does not have more_files feature in claims", () => {
+		// JWT payload: {"more_files": false}
+		const jwt =
+			"header." +
+			Buffer.from(JSON.stringify({ features: ["files"] })).toString("base64") +
+			".signature";
+		expect(isMoreFilesEnabled(jwt)).toBe(false);
+	});
+
+	it("should return false when JWT does not have features claim", () => {
+		// JWT payload: {"sub": "user"}
+		const jwt =
+			"header." +
+			Buffer.from(JSON.stringify({ sub: "user" })).toString("base64") +
+			".signature";
+		expect(isMoreFilesEnabled(jwt)).toBe(false);
+	});
+
+	it("should return false for test tokens without parsing", () => {
+		expect(isMoreFilesEnabled("<<funfetti-auth-jwt>>")).toBe(false);
+		expect(isMoreFilesEnabled("<<funfetti-auth-jwt2>>")).toBe(false);
+		expect(isMoreFilesEnabled("<<aus-completion-token>>")).toBe(false);
+	});
+
+	it("should throw error for invalid JWT format", () => {
+		expect(() => isMoreFilesEnabled("invalid-jwt")).toThrow("Invalid token:");
+	});
+});

packages/wrangler/src/__tests__/pages/project-validate.test.ts

@@ -5,6 +5,7 @@ import { endEventLoop } from "../helpers/end-event-loop";
 import { mockConsoleMethods } from "../helpers/mock-console";
 import { runInTempDir } from "../helpers/run-in-tmp";
 import { runWrangler } from "../helpers/run-wrangler";
+import { validate } from "../../pages/validate";
 
 vi.mock("../../pages/constants", async (importActual) => ({
 	// eslint-disable-next-line @typescript-eslint/consistent-type-imports
@@ -57,4 +58,29 @@ describe("pages project validate", () => {
 			`[Error: Error: Pages only supports up to 10 files in a deployment. Ensure you have specified your build output directory correctly.]`
 		);
 	});
+
+	it("should succeed with custom fileCountLimit even when exceeding default limit", async () => {
+		// Create 11 files, which exceeds the mocked MAX_ASSET_COUNT of 10
+		for (let i = 0; i < 11; i++) {
+			writeFileSync(`logo${i}.png`, Buffer.alloc(1));
+		}
+
+		// Should succeed when passing a custom fileCountLimit of 20
+		const fileMap = await validate({ directory: ".", fileCountLimit: 20 });
+		expect(fileMap.size).toBe(11);
+	});
+
+	it("should error with custom fileCountLimit when exceeding custom limit", async () => {
+		// Create 6 files
+		for (let i = 0; i < 6; i++) {
+			writeFileSync(`logo${i}.png`, Buffer.alloc(1));
+		}
+
+		// Should fail when passing a custom fileCountLimit of 5
+		await expect(() =>
+			validate({ directory: ".", fileCountLimit: 5 })
+		).rejects.toThrowError(
+			"Error: Pages only supports up to 5 files in a deployment. Ensure you have specified your build output directory correctly."
+		);
+	});
 });

packages/wrangler/src/pages/constants.ts

@@ -3,6 +3,7 @@ import { version as wranglerVersion } from "../../package.json";
 const isWindows = process.platform === "win32";
 
 export const MAX_ASSET_COUNT = 20_000;
+export const MAX_ASSET_COUNT_MORE_FILES = 100_000;
 export const MAX_ASSET_SIZE = 25 * 1024 * 1024;
 export const PAGES_CONFIG_CACHE_FILENAME = "pages.json";
 export const MAX_BUCKET_SIZE = 40 * 1024 * 1024;

packages/wrangler/src/pages/upload.ts

@@ -13,6 +13,8 @@ import isInteractive from "../is-interactive";
 import { logger } from "../logger";
 import {
 	BULK_UPLOAD_CONCURRENCY,
+	MAX_ASSET_COUNT,
+	MAX_ASSET_COUNT_MORE_FILES,
 	MAX_BUCKET_FILE_COUNT,
 	MAX_BUCKET_SIZE,
 	MAX_CHECK_MISSING_ATTEMPTS,
@@ -59,7 +61,12 @@ export const pagesProjectUploadCommand = createCommand({
 			throw new FatalError("No JWT given.", 1);
 		}
 
-		const fileMap = await validate({ directory });
+		const fileMap = await validate({
+			directory,
+			fileCountLimit: isMoreFilesEnabled(process.env.CF_PAGES_UPLOAD_JWT)
+				? MAX_ASSET_COUNT_MORE_FILES
+				: MAX_ASSET_COUNT,
+		});
 
 		const manifest = await upload({
 			fileMap,
@@ -400,6 +407,38 @@ export const isJwtExpired = (token: string): boolean | undefined => {
 	}
 };
 
+export const isMoreFilesEnabled = (token: string): boolean => {
+	// During testing we don't use valid JWTs, so don't try and parse them
+	if (
+		typeof vitest !== "undefined" &&
+		(token === "<<funfetti-auth-jwt>>" ||
+			token === "<<funfetti-auth-jwt2>>" ||
+			token === "<<aus-completion-token>>")
+	) {
+		return false;
+	}
+	try {
+		// Not validating the JWT here, which ordinarily would be a big red flag.
+		// However, if the JWT is invalid, no uploads (calls to /pages/assets/upload)
+		// will succeed.
+		const decodedJwt = JSON.parse(
+			Buffer.from(token.split(".")[1], "base64").toString()
+		);
+
+		const features = decodedJwt.features as Array<string>;
+		if (!features || !Array.isArray(features)) {
+			return false;
+		}
+
+		return features.some((feat) => feat === "more_files");
+	} catch (e) {
+		if (e instanceof Error) {
+			throw new Error(`Invalid token: ${e.message}`);
+		}
+		return false;
+	}
+};
+
 function formatTime(duration: number) {
 	return `(${(duration / 1000).toFixed(2)} sec)`;
 }

packages/wrangler/src/pages/validate.ts

@@ -46,6 +46,7 @@ export type FileContainer = {
 
 export const validate = async (args: {
 	directory: string;
+	fileCountLimit?: number;
 }): Promise<Map<string, FileContainer>> => {
 	const IGNORE_LIST = [
 		"_worker.js",
@@ -68,6 +69,8 @@ export const validate = async (args: {
 	// 	maxMemory = (parsed['max-old-space-size'] ? parsed['max-old-space-size'] : parsed['max_old_space_size']) * 1000 * 1000; // Turn MB into bytes
 	// }
 
+	const fileCountLimit = args.fileCountLimit ?? MAX_ASSET_COUNT;
+
 	const walk = async (
 		dir: string,
 		fileMap: Map<string, FileContainer> = new Map(),
@@ -124,9 +127,9 @@ export const validate = async (args: {
 
 	const fileMap = await walk(directory);
 
-	if (fileMap.size > MAX_ASSET_COUNT) {
+	if (fileMap.size > fileCountLimit) {
 		throw new FatalError(
-			`Error: Pages only supports up to ${MAX_ASSET_COUNT.toLocaleString()} files in a deployment. Ensure you have specified your build output directory correctly.`,
+			`Error: Pages only supports up to ${fileCountLimit.toLocaleString()} files in a deployment. Ensure you have specified your build output directory correctly.`,
 			1
 		);
 	}