perf(analytics): introduce caching for analytics data via supabase

packages/fern-dashboard/package.json

@@ -77,6 +77,7 @@
     "@react-three/fiber": "9.0.4",
     "@sentry/nextjs": "^10.12.0",
     "@sparticuz/chromium": "^133.0.0",
+    "@supabase/supabase-js": "^2.84.0",
     "@tailwindcss/container-queries": "^0.1.1",
     "@tailwindcss/typography": "^0.5.16",
     "@tanstack/react-query": "^5.71.1",
@@ -105,6 +106,7 @@
     "@types/uuid": "^9.0.8",
     "@upstash/redis": "^1.34.6",
     "@vercel/analytics": "^1.5.0",
+    "@vercel/kv": "^3.0.0",
     "@vercel/speed-insights": "^1.2.0",
     "algoliasearch": "^5.40.1",
     "archiver": "^7.0.1",
@@ -132,6 +134,8 @@
     "motion": "^12.4.7",
     "next": "catalog:",
     "next-themes": "^0.4.4",
+    "p-limit": "^7.2.0",
+    "pg": "^8.16.3",
     "posthog-js": "^1.258.2",
     "posthog-node": "^5.10.0",
     "puppeteer-core": "^24.4.0",

packages/fern-dashboard/scripts/list-production-domains.ts

@@ -0,0 +1,102 @@
+#!/usr/bin/env npx tsx
+/**
+ * List all production domains from the KV store
+ *
+ * Usage:
+ *   npx tsx scripts/list-production-domains.ts
+ *   npx tsx scripts/list-production-domains.ts --include-previews
+ *   npx tsx scripts/list-production-domains.ts --json
+ */
+
+import { config } from "dotenv";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { parseArgs } from "util";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+config({ path: resolve(__dirname, "../.env.local") });
+
+process.env.NODE_ENV = "test";
+
+async function main() {
+    const { values } = parseArgs({
+        options: {
+            "include-previews": { type: "boolean", default: false },
+            json: { type: "boolean", default: false },
+            help: { type: "boolean", short: "h" }
+        },
+        allowPositionals: false
+    });
+
+    if (values.help) {
+        console.log(`
+List Production Domains
+
+Usage:
+  npx tsx scripts/list-production-domains.ts [options]
+
+Options:
+  --include-previews  Include Fern-hosted preview domains (*.docs.buildwithfern.com)
+  --json              Output as JSON
+  -h, --help          Show this help message
+        `);
+        process.exit(0);
+    }
+
+    // Check for either KV (with CDN_URI) or FDR credentials
+    const hasKV = process.env.KV_REST_API_URL && process.env.KV_REST_API_TOKEN && process.env.NEXT_PUBLIC_CDN_URI;
+    const hasFDR = (process.env.FDR_SERVER_URL || process.env.NEXT_PUBLIC_FDR_ORIGIN) && process.env.FERN_TOKEN;
+
+    if (!hasKV && !hasFDR) {
+        console.error("Error: Either KV credentials (KV_REST_API_URL, KV_REST_API_TOKEN, NEXT_PUBLIC_CDN_URI)");
+        console.error("       or FDR credentials (FDR_SERVER_URL/NEXT_PUBLIC_FDR_ORIGIN, FERN_TOKEN) are required");
+        process.exit(1);
+    }
+
+    console.log(`Using ${hasKV ? "KV store" : "FDR"} to fetch domains...\n`);
+
+    const { getAllProductionDomains, getAllDomainsIncludingPreviews } = await import(
+        "../src/app/services/analyticsCron/getAllProductionDomains"
+    );
+
+    try {
+        if (values["include-previews"]) {
+            const domains = await getAllDomainsIncludingPreviews();
+
+            if (values.json) {
+                console.log(JSON.stringify(domains, null, 2));
+            } else {
+                console.log(`\nFound ${domains.length} domains (including previews):\n`);
+                for (const domain of domains) {
+                    console.log(`  ${domain}`);
+                }
+            }
+        } else {
+            const domains = await getAllProductionDomains();
+
+            if (values.json) {
+                console.log(JSON.stringify(domains, null, 2));
+            } else {
+                const customDomains = domains.filter((d) => d.isCustomDomain);
+                const fernDomains = domains.filter((d) => !d.isCustomDomain);
+
+                console.log(`\nFound ${domains.length} production domains:\n`);
+                console.log(`Custom domains (${customDomains.length}):`);
+                for (const d of customDomains) {
+                    console.log(`  ${d.domain}`);
+                }
+                console.log(`\nFern-hosted domains (${fernDomains.length}):`);
+                for (const d of fernDomains) {
+                    console.log(`  ${d.domain}`);
+                }
+            }
+        }
+    } catch (error) {
+        console.error("Error fetching domains:", error);
+        process.exit(1);
+    }
+}
+
+main();

packages/fern-dashboard/scripts/profile-analytics-queries.ts

@@ -0,0 +1,172 @@
+#!/usr/bin/env npx tsx
+/**
+ * Profile PostHog Analytics Queries
+ *
+ * Profiles each PostHog query and logs timing data to a JSON file via the service.
+ *
+ * Usage:
+ *   npx tsx scripts/profile-analytics-queries.ts --org-name=fern --site=buildwithfern.com/learn --period=7
+ *   npx tsx scripts/profile-analytics-queries.ts --org-name=fern --period=30
+ */
+
+// Set test env FIRST to skip server-only checks
+process.env.NODE_ENV = "test";
+
+import { config } from "dotenv";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+config({ path: resolve(__dirname, "../.env.local") });
+
+import { parseArgs } from "util";
+
+type DateRangePeriod = 7 | 14 | 30 | 90 | 180;
+
+async function profileSite(
+    docsSite: string,
+    docsOrg: string,
+    period: DateRangePeriod,
+    additionalDomains: string[] = []
+): Promise<void> {
+    const { AnalyticsService } = await import("../src/app/services/posthog/analytics");
+    const { setProfilingContext } = await import("../src/app/services/posthog/client");
+
+    setProfilingContext({ org: docsOrg, docs_site: docsSite, range_days: period });
+
+    const analytics = new AnalyticsService({
+        userId: "profiler",
+        baseSiteUrl: docsSite,
+        additionalDomains
+    });
+
+    const dateRange = { type: "last_n_days" as const, days: period };
+
+    console.log(`  Profiling ${docsSite}...`);
+
+    const queryConfigs = [
+        { name: "getTopPages", fn: () => analytics.getTopPages({ dateRange, limit: 10 }) },
+        { name: "getTopCountries", fn: () => analytics.getTopCountries({ dateRange, limit: 10 }) },
+        { name: "getChannels", fn: () => analytics.getChannels({ dateRange, limit: 10 }) },
+        { name: "getDeviceTypes", fn: () => analytics.getDeviceTypes({ dateRange, limit: 10 }) },
+        { name: "getReferringDomains", fn: () => analytics.getReferringDomains({ dateRange, limit: 10 }) },
+        { name: "getLLMFileViews", fn: () => analytics.getLLMFileViews({ dateRange, limit: 10 }) },
+        { name: "getAPIExplorerRequests", fn: () => analytics.getAPIExplorerRequests({ dateRange, limit: 20 }) },
+        { name: "getLLMBotTrafficByProvider", fn: () => analytics.getLLMBotTrafficByProvider({ dateRange, limit: 10 }) }
+    ];
+
+    for (const { name, fn } of queryConfigs) {
+        const start = performance.now();
+        try {
+            await fn();
+            const duration = Math.round(performance.now() - start);
+            console.log(`    ✓ ${name}: ${duration}ms`);
+        } catch (err) {
+            const duration = Math.round(performance.now() - start);
+            console.log(`    ✗ ${name}: ${duration}ms (error)`);
+        }
+    }
+}
+
+async function main() {
+    const { values } = parseArgs({
+        options: {
+            "org-id": { type: "string" },
+            "org-name": { type: "string" },
+            site: { type: "string" },
+            period: { type: "string", default: "7" },
+            output: { type: "string", short: "o" },
+            help: { type: "boolean", short: "h" }
+        },
+        allowPositionals: false
+    });
+
+    if (values.help) {
+        console.log(`
+Profile PostHog Analytics Queries
+
+Usage:
+  npx tsx scripts/profile-analytics-queries.ts [options]
+
+Options:
+  --org-id <id>       Filter by organization ID
+  --org-name <name>   Filter by organization name
+  --site <domain>     Specific docs site domain
+  --period <days>     Date range: 7, 14, 30, 90, or 180 (default: 7)
+  -o, --output <file> Output JSON file (default: analytics-profile-{timestamp}.json)
+  -h, --help          Show this help message
+
+Examples:
+  npx tsx scripts/profile-analytics-queries.ts --org-name=fern --site=buildwithfern.com/learn --period=7
+  npx tsx scripts/profile-analytics-queries.ts --org-name=fern --period=30 -o profile.json
+        `);
+        process.exit(0);
+    }
+
+    const periodNum = parseInt(values.period || "7", 10);
+    const validPeriods = [7, 14, 30, 90, 180];
+    if (!validPeriods.includes(periodNum)) {
+        console.error(`Invalid period: ${periodNum}. Must be one of: ${validPeriods.join(", ")}`);
+        process.exit(1);
+    }
+    const period = periodNum as DateRangePeriod;
+
+    const requiredEnvVars = ["POSTHOG_ANALYTICS_PROJECT_ID", "POSTHOG_ANALYTICS_API_KEY"];
+    for (const envVar of requiredEnvVars) {
+        if (!process.env[envVar]) {
+            console.error(`Error: ${envVar} environment variable is required`);
+            process.exit(1);
+        }
+    }
+
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+    const outputFile = values.output || resolve(__dirname, `../analytics-profile-${timestamp}.json`);
+
+    const { enableProfiling, disableProfiling } = await import("../src/app/services/posthog/client");
+
+    enableProfiling(outputFile);
+
+    try {
+        if (values.site) {
+            const orgName = values["org-name"] || values["org-id"] || "unknown";
+            console.log(`\nProfiling site: ${values.site} (org: ${orgName}, period: ${period} days)\n`);
+            await profileSite(values.site, orgName, period);
+        } else if (values["org-name"] || values["org-id"]) {
+            // Use KV to get all production domains, then filter by org if needed
+            const { getAllProductionDomains } = await import(
+                "../src/app/services/analyticsCron/getAllProductionDomains"
+            );
+            const orgFilter = values["org-id"] || values["org-name"] || "";
+
+            console.log(`\nFetching production domains from KV...`);
+            const allDomains = await getAllProductionDomains();
+
+            // Filter by org if orgId is available in the domain data
+            const orgSites = orgFilter
+                ? allDomains.filter((d) => d.orgId === orgFilter || d.domain.includes(orgFilter))
+                : allDomains;
+
+            console.log(
+                `Found ${orgSites.length} sites for org "${orgFilter}". Profiling each (period: ${period} days)...\n`
+            );
+
+            for (const site of orgSites) {
+                await profileSite(site.domain, site.orgId || orgFilter, period);
+            }
+        } else {
+            console.error("Error: Must specify --site, --org-name, or --org-id");
+            process.exit(1);
+        }
+    } finally {
+        disableProfiling();
+    }
+
+    console.log(`\nProfile saved to: ${outputFile}`);
+}
+
+main().catch((err) => {
+    console.error("Fatal error:", err);
+    process.exit(1);
+});