From 62803a445ce09f0e988daa2e084a405dc061cb37 Mon Sep 17 00:00:00 2001 From: yfe404 Date: Thu, 27 Nov 2025 16:12:35 +0100 Subject: [PATCH 01/18] feat: add Apify integration for running Actors Adds a new "Run Actor" action that allows workflows to execute Apify Actors. Features: - Integration setup with API token configuration - Actor selection by ID (e.g., apify/rag-web-browser) - JSON input support for Actor parameters - Sync/async execution modes - Full input/output display in Runs panel - Connection testing Files added: - plugins/apify/ - Complete plugin implementation Files modified: - action-grid.tsx, action-config.tsx - UI for Run Actor action - workflow-executor.workflow.ts - Runtime execution handler - credential-fetcher.ts - API key mapping - integration-form-dialog.tsx - Settings form - test/route.ts - Connection test endpoint Limitations: - Icon is a placeholder, not the official Apify logo --- .../[integrationId]/test/route.ts | 48 ++++++++++ .../settings/integration-form-dialog.tsx | 26 +++++ components/ui/integration-icon.tsx | 31 +++++- components/workflow/config/action-config.tsx | 92 ++++++++++++++++++ components/workflow/config/action-grid.tsx | 11 ++- components/workflow/node-config-panel.tsx | 1 + components/workflow/nodes/action-node.tsx | 1 + lib/credential-fetcher.ts | 12 +++ lib/db/integrations.ts | 5 +- lib/db/schema.ts | 2 +- lib/steps/index.ts | 5 + lib/workflow-executor.workflow.ts | 23 +++++ plugins/apify/codegen/run-actor.ts | 53 +++++++++++ plugins/apify/icon.tsx | 16 ++++ plugins/apify/index.tsx | 68 +++++++++++++ plugins/apify/settings.tsx | 47 +++++++++ plugins/apify/steps/run-actor/config.tsx | 85 +++++++++++++++++ plugins/apify/steps/run-actor/step.ts | 95 +++++++++++++++++++ plugins/apify/test.ts | 22 +++++ plugins/index.ts | 3 +- 20 files changed, 641 insertions(+), 5 deletions(-) create mode 100644 plugins/apify/codegen/run-actor.ts create mode 100644 plugins/apify/icon.tsx create mode 100644 plugins/apify/index.tsx create mode 100644 plugins/apify/settings.tsx create mode 100644 plugins/apify/steps/run-actor/config.tsx create mode 100644 plugins/apify/steps/run-actor/step.ts create mode 100644 plugins/apify/test.ts diff --git a/app/api/integrations/[integrationId]/test/route.ts b/app/api/integrations/[integrationId]/test/route.ts index 0247e731..f4add1fb 100644 --- a/app/api/integrations/[integrationId]/test/route.ts +++ b/app/api/integrations/[integrationId]/test/route.ts @@ -68,6 +68,9 @@ export async function POST( integration.config.firecrawlApiKey ); break; + case "apify": + result = await testApifyConnection(integration.config.apifyApiKey); + break; default: return NextResponse.json( { error: "Invalid integration type" }, @@ -281,3 +284,48 @@ async function testFirecrawlConnection( }; } } + +async function testApifyConnection( + apiKey?: string +): Promise { + try { + if (!apiKey) { + return { + status: "error", + message: "Apify API Token is not configured", + }; + } + + // Test by fetching user info from Apify API + const response = await fetch("https://api.apify.com/v2/users/me", { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + + if (!response.ok) { + return { + status: "error", + message: "Invalid API token", + }; + } + + const data = await response.json(); + if (!data.data?.username) { + return { + status: "error", + message: "Failed to verify API token", + }; + } + + return { + status: "success", + message: `Connected as ${data.data.username}`, + }; + } catch (error) { + return { + status: "error", + message: error instanceof Error ? error.message : "Connection failed", + }; + } +} diff --git a/components/settings/integration-form-dialog.tsx b/components/settings/integration-form-dialog.tsx index f78e70be..4bdfbe9f 100644 --- a/components/settings/integration-form-dialog.tsx +++ b/components/settings/integration-form-dialog.tsx @@ -41,6 +41,7 @@ type IntegrationFormData = { const INTEGRATION_TYPES: IntegrationType[] = [ "ai-gateway", + "apify", "database", "linear", "resend", @@ -55,6 +56,7 @@ const INTEGRATION_LABELS: Record = { database: "Database", "ai-gateway": "AI Gateway", firecrawl: "Firecrawl", + apify: "Apify", }; export function IntegrationFormDialog({ @@ -290,6 +292,30 @@ export function IntegrationFormDialog({

); + case "apify": + return ( +
+ + updateConfig("apifyApiKey", e.target.value)} + placeholder="apify_api_..." + type="password" + value={formData.config.apifyApiKey || ""} + /> +

+ Get your API token from{" "} + + Apify Console + +

+
+ ); default: return null; } diff --git a/components/ui/integration-icon.tsx b/components/ui/integration-icon.tsx index a53154a0..fbec4482 100644 --- a/components/ui/integration-icon.tsx +++ b/components/ui/integration-icon.tsx @@ -3,7 +3,7 @@ import Image from "next/image"; import { cn } from "@/lib/utils"; interface IntegrationIconProps { - integration: "linear" | "resend" | "slack" | "vercel" | "database" | "ai-gateway" | "firecrawl"; + integration: "linear" | "resend" | "slack" | "vercel" | "database" | "ai-gateway" | "firecrawl" | "apify"; className?: string; } @@ -56,6 +56,31 @@ function FirecrawlIcon({ className }: { className?: string }) { ); } +function ApifyIcon({ className }: { className?: string }) { + return ( + + + + + + ); +} + export function IntegrationIcon({ integration, className = "h-3 w-3", @@ -86,6 +111,10 @@ export function IntegrationIcon({ return ; } + if (integration === "apify") { + return ; + } + return ( {`${integration}; + onUpdateConfig: (key: string, value: string) => void; + disabled: boolean; +}) { + return ( + <> +
+ + onUpdateConfig("actorId", value)} + placeholder="apify/web-scraper or {{NodeName.actorId}}" + value={(config?.actorId as string) || ""} + /> +

+ Enter the Actor ID (e.g., apify/web-scraper) or use a template + reference. +

+
+
+ + { + try { + JSON.parse(value); + onUpdateConfig("actorInput", value); + } catch { + // Store as string if not valid JSON yet (user is still typing) + onUpdateConfig("actorInputRaw", value); + } + }} + placeholder='{"startUrls": [{"url": "https://example.com"}]}' + rows={6} + value={ + (config?.actorInputRaw as string) || + (config?.actorInput + ? JSON.stringify(config.actorInput, null, 2) + : "") + } + /> +

+ JSON input for the Actor. Check the Actor's documentation for required + fields. +

+
+
+ onUpdateConfig("waitForFinish", e.target.checked ? "true" : "false")} + className="h-4 w-4 rounded border-input" + /> +
+ +

+ Wait for the Actor to finish and return dataset items +

+
+
+ + ); +} + // Action categories and their actions const ACTION_CATEGORIES = { System: ["HTTP Request", "Database Query", "Condition"], "AI Gateway": ["Generate Text", "Generate Image"], + Apify: ["Run Actor"], Firecrawl: ["Scrape", "Search"], Linear: ["Create Ticket", "Find Issues"], Resend: ["Send Email"], @@ -734,6 +811,12 @@ export function ActionConfig({ AI Gateway + +
+ + Apify +
+
@@ -884,6 +967,15 @@ export function ActionConfig({ onUpdateConfig={onUpdateConfig} /> )} + + {/* Run Actor fields (Apify) */} + {config?.actionType === "Run Actor" && ( + + )} ); } diff --git a/components/workflow/config/action-grid.tsx b/components/workflow/config/action-grid.tsx index a6268aae..29fce4e4 100644 --- a/components/workflow/config/action-grid.tsx +++ b/components/workflow/config/action-grid.tsx @@ -5,6 +5,7 @@ import { Flame, Mail, MessageSquare, + Play, Search, Settings, Sparkles, @@ -23,7 +24,7 @@ type ActionType = { description: string; category: string; icon: React.ComponentType<{ className?: string }>; - integration?: "linear" | "resend" | "slack" | "vercel" | "firecrawl"; + integration?: "linear" | "resend" | "slack" | "vercel" | "firecrawl" | "apify"; }; const actions: ActionType[] = [ @@ -112,6 +113,14 @@ const actions: ActionType[] = [ icon: Search, integration: "firecrawl", }, + { + id: "Run Actor", + label: "Run Actor", + description: "Run an Apify Actor", + category: "Apify", + icon: Play, + integration: "apify", + }, ]; type ActionGridProps = { diff --git a/components/workflow/node-config-panel.tsx b/components/workflow/node-config-panel.tsx index 2c7dd3d4..11650dca 100644 --- a/components/workflow/node-config-panel.tsx +++ b/components/workflow/node-config-panel.tsx @@ -665,6 +665,7 @@ export const PanelInner = () => { "Database Query": "database", Scrape: "firecrawl", Search: "firecrawl", + "Run Actor": "apify", } as const; const integrationType = diff --git a/components/workflow/nodes/action-node.tsx b/components/workflow/nodes/action-node.tsx index 728880f4..13ca7724 100644 --- a/components/workflow/nodes/action-node.tsx +++ b/components/workflow/nodes/action-node.tsx @@ -79,6 +79,7 @@ const getIntegrationFromActionType = (actionType: string): string => { Scrape: "Firecrawl", Search: "Firecrawl", Condition: "Condition", + "Run Actor": "Apify", }; return integrationMap[actionType] || "System"; }; diff --git a/lib/credential-fetcher.ts b/lib/credential-fetcher.ts index 5d8745db..19283e05 100644 --- a/lib/credential-fetcher.ts +++ b/lib/credential-fetcher.ts @@ -26,6 +26,7 @@ export type WorkflowCredentials = { AI_GATEWAY_API_KEY?: string; DATABASE_URL?: string; FIRECRAWL_API_KEY?: string; + APIFY_API_KEY?: string; }; function mapResendConfig(config: IntegrationConfig): WorkflowCredentials { @@ -82,6 +83,14 @@ function mapFirecrawlConfig(config: IntegrationConfig): WorkflowCredentials { return creds; } +function mapApifyConfig(config: IntegrationConfig): WorkflowCredentials { + const creds: WorkflowCredentials = {}; + if (config.apifyApiKey) { + creds.APIFY_API_KEY = config.apifyApiKey; + } + return creds; +} + /** * Map integration config to WorkflowCredentials format */ @@ -107,6 +116,9 @@ function mapIntegrationConfig( if (integrationType === "firecrawl") { return mapFirecrawlConfig(config); } + if (integrationType === "apify") { + return mapApifyConfig(config); + } return {}; } diff --git a/lib/db/integrations.ts b/lib/db/integrations.ts index 26827196..2bec604a 100644 --- a/lib/db/integrations.ts +++ b/lib/db/integrations.ts @@ -100,7 +100,8 @@ export type IntegrationType = | "slack" | "database" | "ai-gateway" - | "firecrawl"; + | "firecrawl" + | "apify"; export type IntegrationConfig = { // Resend @@ -115,6 +116,8 @@ export type IntegrationConfig = { openaiApiKey?: string; // Firecrawl firecrawlApiKey?: string; + // Apify + apifyApiKey?: string; }; export type DecryptedIntegration = { diff --git a/lib/db/schema.ts b/lib/db/schema.ts index cb6d7056..57d11683 100644 --- a/lib/db/schema.ts +++ b/lib/db/schema.ts @@ -85,7 +85,7 @@ export const integrations = pgTable("integrations", { type: text("type") .notNull() .$type< - "resend" | "linear" | "slack" | "database" | "ai-gateway" | "firecrawl" + "resend" | "linear" | "slack" | "database" | "ai-gateway" | "firecrawl" | "apify" >(), // biome-ignore lint/suspicious/noExplicitAny: JSONB type - encrypted credentials stored as JSON config: jsonb("config").notNull().$type(), diff --git a/lib/steps/index.ts b/lib/steps/index.ts index 76921efa..b0dbd25a 100644 --- a/lib/steps/index.ts +++ b/lib/steps/index.ts @@ -6,6 +6,7 @@ import type { generateImageStep } from "../../plugins/ai-gateway/steps/generate-image/step"; import type { generateTextStep } from "../../plugins/ai-gateway/steps/generate-text/step"; +import type { apifyRunActorStep } from "../../plugins/apify/steps/run-actor/step"; import type { firecrawlScrapeStep } from "../../plugins/firecrawl/steps/scrape/step"; import type { firecrawlSearchStep } from "../../plugins/firecrawl/steps/search/step"; import type { createTicketStep } from "../../plugins/linear/steps/create-ticket/step"; @@ -73,6 +74,10 @@ export const stepRegistry: Record = { ( await import("../../plugins/firecrawl/steps/search/step") ).firecrawlSearchStep(input as Parameters[0]), + "Run Actor": async (input) => + ( + await import("../../plugins/apify/steps/run-actor/step") + ).apifyRunActorStep(input as Parameters[0]), }; // Helper to check if a step exists diff --git a/lib/workflow-executor.workflow.ts b/lib/workflow-executor.workflow.ts index 99dd3b0b..3847a405 100644 --- a/lib/workflow-executor.workflow.ts +++ b/lib/workflow-executor.workflow.ts @@ -212,6 +212,29 @@ async function executeActionStep(input: { return await firecrawlSearchStep(stepInput as any); } + if (actionType === "Run Actor") { + const { apifyRunActorStep } = await import( + "../plugins/apify/steps/run-actor/step" + ); + // Parse actorInput if it's a JSON string and convert waitForFinish to boolean + const runActorInput = { ...stepInput }; + if (typeof runActorInput.actorInput === "string") { + try { + runActorInput.actorInput = JSON.parse(runActorInput.actorInput); + } catch { + // If JSON parsing fails, keep the original string + console.warn("[Run Actor] Failed to parse actorInput as JSON"); + } + } + // Convert waitForFinish string to boolean (checkbox saves as "true"/"false" strings) + if (typeof runActorInput.waitForFinish === "string") { + runActorInput.waitForFinish = runActorInput.waitForFinish === "true"; + } + console.log("[Run Actor] Input:", JSON.stringify(runActorInput, null, 2)); + // biome-ignore lint/suspicious/noExplicitAny: Dynamic step input type + return await apifyRunActorStep(runActorInput as any); + } + // Fallback for unknown action types return { success: false, diff --git a/plugins/apify/codegen/run-actor.ts b/plugins/apify/codegen/run-actor.ts new file mode 100644 index 00000000..811a738e --- /dev/null +++ b/plugins/apify/codegen/run-actor.ts @@ -0,0 +1,53 @@ +/** + * Code generation template for Apify Run Actor action + * This template is used when exporting workflows to standalone Next.js projects + * It uses environment variables instead of integrationId + */ +export const runActorCodegenTemplate = `const APIFY_API_BASE = "https://api.apify.com/v2"; + +export async function apifyRunActorStep(input: { + actorId: string; + actorInput?: Record; + waitForFinish?: boolean; + maxWaitSecs?: number; +}) { + "use step"; + + const apiKey = process.env.APIFY_API_KEY!; + const waitForFinish = input.waitForFinish !== false; + const maxWaitSecs = input.maxWaitSecs || 120; + + const runUrl = waitForFinish + ? \`\${APIFY_API_BASE}/acts/\${encodeURIComponent(input.actorId)}/run-sync-get-dataset-items?timeout=\${maxWaitSecs}\` + : \`\${APIFY_API_BASE}/acts/\${encodeURIComponent(input.actorId)}/runs\`; + + const runResponse = await fetch(runUrl, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: \`Bearer \${apiKey}\`, + }, + body: JSON.stringify(input.actorInput || {}), + }); + + if (!runResponse.ok) { + const errorText = await runResponse.text().catch(() => "Unknown error"); + throw new Error(\`Failed to run Actor: \${runResponse.status} - \${errorText}\`); + } + + if (waitForFinish) { + const data = await runResponse.json(); + return { + runId: runResponse.headers.get("x-apify-run-id") || "unknown", + status: "SUCCEEDED", + data: Array.isArray(data) ? data : [data], + }; + } + + const runData = await runResponse.json(); + return { + runId: runData.data?.id || "unknown", + status: runData.data?.status || "RUNNING", + datasetId: runData.data?.defaultDatasetId, + }; +}`; diff --git a/plugins/apify/icon.tsx b/plugins/apify/icon.tsx new file mode 100644 index 00000000..41b95bc8 --- /dev/null +++ b/plugins/apify/icon.tsx @@ -0,0 +1,16 @@ +export function ApifyIcon({ className }: { className?: string }) { + return ( + + Apify + + + ); +} diff --git a/plugins/apify/index.tsx b/plugins/apify/index.tsx new file mode 100644 index 00000000..45bb4f84 --- /dev/null +++ b/plugins/apify/index.tsx @@ -0,0 +1,68 @@ +import { Play } from "lucide-react"; +import type { IntegrationPlugin } from "../registry"; +import { registerIntegration } from "../registry"; +import { runActorCodegenTemplate } from "./codegen/run-actor"; +import { ApifyIcon } from "./icon"; +import { ApifySettings } from "./settings"; +import { RunActorConfigFields } from "./steps/run-actor/config"; +import { testApify } from "./test"; + +const apifyPlugin: IntegrationPlugin = { + type: "apify", + label: "Apify", + description: "Run web scraping and automation Actors", + + icon: { + type: "svg", + value: "ApifyIcon", + svgComponent: ApifyIcon, + }, + + settingsComponent: ApifySettings, + + formFields: [ + { + id: "apifyApiKey", + label: "API Token", + type: "password", + placeholder: "apify_api_...", + configKey: "apifyApiKey", + helpText: "Get your API token from ", + helpLink: { + text: "Apify Console", + url: "https://console.apify.com/account/integrations", + }, + }, + ], + + credentialMapping: (config) => { + const creds: Record = {}; + if (config.apifyApiKey) { + creds.APIFY_API_KEY = String(config.apifyApiKey); + } + return creds; + }, + + testConfig: { + testFunction: testApify, + }, + + actions: [ + { + id: "Run Actor", + label: "Run Actor", + description: "Run an Apify Actor and get results", + category: "Apify", + icon: Play, + stepFunction: "apifyRunActorStep", + stepImportPath: "run-actor", + configFields: RunActorConfigFields, + codegenTemplate: runActorCodegenTemplate, + }, + ], +}; + +// Auto-register on import +registerIntegration(apifyPlugin); + +export default apifyPlugin; diff --git a/plugins/apify/settings.tsx b/plugins/apify/settings.tsx new file mode 100644 index 00000000..932f9a56 --- /dev/null +++ b/plugins/apify/settings.tsx @@ -0,0 +1,47 @@ +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; + +export function ApifySettings({ + apiKey, + hasKey, + onApiKeyChange, +}: { + apiKey: string; + hasKey?: boolean; + onApiKeyChange: (key: string) => void; + showCard?: boolean; + config?: Record; + onConfigChange?: (key: string, value: string) => void; +}) { + return ( +
+
+ + onApiKeyChange(e.target.value)} + placeholder={ + hasKey ? "API token is configured" : "Enter your Apify API token" + } + type="password" + value={apiKey} + /> +

+ Get your API token from{" "} + + Apify Console + + . +

+
+
+ ); +} diff --git a/plugins/apify/steps/run-actor/config.tsx b/plugins/apify/steps/run-actor/config.tsx new file mode 100644 index 00000000..c14f668c --- /dev/null +++ b/plugins/apify/steps/run-actor/config.tsx @@ -0,0 +1,85 @@ +import { Label } from "@/components/ui/label"; +import { TemplateBadgeInput } from "@/components/ui/template-badge-input"; +import { TemplateBadgeTextarea } from "@/components/ui/template-badge-textarea"; + +/** + * Run Actor Config Fields Component + * UI for configuring the run actor action + */ +export function RunActorConfigFields({ + config, + onUpdateConfig, + disabled, +}: { + config: Record; + onUpdateConfig: (key: string, value: unknown) => void; + disabled?: boolean; +}) { + return ( +
+
+ + onUpdateConfig("actorId", value)} + placeholder="apify/web-scraper or {{NodeName.actorId}}" + value={(config?.actorId as string) || ""} + /> +

+ Enter the Actor ID (e.g., apify/web-scraper) or use a template + reference. +

+
+ +
+ + { + try { + const parsed = JSON.parse(value); + onUpdateConfig("actorInput", parsed); + onUpdateConfig("actorInputRaw", value); + } catch { + // Store as string if not valid JSON yet (user is still typing) + onUpdateConfig("actorInputRaw", value); + } + }} + placeholder='{"startUrls": [{"url": "https://example.com"}]}' + rows={6} + value={ + (config?.actorInputRaw as string) || + (config?.actorInput + ? JSON.stringify(config.actorInput, null, 2) + : "") + } + /> +

+ JSON input for the Actor. Check the Actor's documentation for required + fields. +

+
+ +
+ onUpdateConfig("waitForFinish", e.target.checked)} + className="h-4 w-4 rounded border-input" + /> +
+ +

+ Wait for the Actor to finish and return dataset items +

+
+
+
+ ); +} diff --git a/plugins/apify/steps/run-actor/step.ts b/plugins/apify/steps/run-actor/step.ts new file mode 100644 index 00000000..e23a6585 --- /dev/null +++ b/plugins/apify/steps/run-actor/step.ts @@ -0,0 +1,95 @@ +import "server-only"; + +import { fetchCredentials } from "@/lib/credential-fetcher"; +import { getErrorMessage } from "@/lib/utils"; + +const APIFY_API_BASE = "https://api.apify.com/v2"; + +type ApifyRunActorResult = + | { + success: true; + runId: string; + status: string; + datasetId?: string; + data?: unknown[]; + } + | { success: false; error: string }; + +/** + * Apify Run Actor Step + * Runs an Apify Actor and optionally waits for results + */ +export async function apifyRunActorStep(input: { + integrationId?: string; + actorId: string; + actorInput?: Record; + waitForFinish?: boolean; + maxWaitSecs?: number; +}): Promise { + "use step"; + + const credentials = input.integrationId + ? await fetchCredentials(input.integrationId) + : {}; + + const apiKey = credentials.APIFY_API_KEY; + + if (!apiKey) { + return { + success: false, + error: "Apify API Token is not configured.", + }; + } + + try { + const waitForFinish = input.waitForFinish !== false; + const maxWaitSecs = input.maxWaitSecs || 120; + + // Start the Actor run + const runUrl = waitForFinish + ? `${APIFY_API_BASE}/acts/${encodeURIComponent(input.actorId)}/run-sync-get-dataset-items?timeout=${maxWaitSecs}` + : `${APIFY_API_BASE}/acts/${encodeURIComponent(input.actorId)}/runs`; + + const runResponse = await fetch(runUrl, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify(input.actorInput || {}), + }); + + if (!runResponse.ok) { + const errorText = await runResponse.text().catch(() => "Unknown error"); + return { + success: false, + error: `Failed to run Actor: ${runResponse.status} - ${errorText}`, + }; + } + + if (waitForFinish) { + // For sync runs, we get the dataset items directly + const data = await runResponse.json(); + return { + success: true, + runId: runResponse.headers.get("x-apify-run-id") || "unknown", + status: "SUCCEEDED", + data: Array.isArray(data) ? data : [data], + }; + } + + // For async runs, we get the run info + const runData = await runResponse.json(); + return { + success: true, + runId: runData.data?.id || "unknown", + status: runData.data?.status || "RUNNING", + datasetId: runData.data?.defaultDatasetId, + }; + } catch (error) { + return { + success: false, + error: `Failed to run Actor: ${getErrorMessage(error)}`, + }; + } +} diff --git a/plugins/apify/test.ts b/plugins/apify/test.ts new file mode 100644 index 00000000..681a516b --- /dev/null +++ b/plugins/apify/test.ts @@ -0,0 +1,22 @@ +export async function testApify(credentials: Record) { + try { + // Test the API token by fetching user info + const response = await fetch("https://api.apify.com/v2/users/me", { + method: "GET", + headers: { + Authorization: `Bearer ${credentials.APIFY_API_KEY}`, + }, + }); + + if (response.ok) { + return { success: true }; + } + const error = await response.text(); + return { success: false, error }; + } catch (error) { + return { + success: false, + error: error instanceof Error ? error.message : String(error), + }; + } +} diff --git a/plugins/index.ts b/plugins/index.ts index 3b4e732d..285029f1 100644 --- a/plugins/index.ts +++ b/plugins/index.ts @@ -13,10 +13,11 @@ * 1. Delete the plugin directory * 2. Run: pnpm discover-plugins (or it runs automatically on build) * - * Discovered plugins: ai-gateway, firecrawl, linear, resend, slack + * Discovered plugins: ai-gateway, apify, firecrawl, linear, resend, slack */ import "./ai-gateway"; +import "./apify"; import "./firecrawl"; import "./linear"; import "./resend"; From 1cb415d8481cc98e2eef56d5ae6df3393ee523c7 Mon Sep 17 00:00:00 2001 From: yfe404 Date: Thu, 27 Nov 2025 16:21:43 +0100 Subject: [PATCH 02/18] fix: lint and formatting issues --- components/workflow/config/action-config.tsx | 11 +++++++---- components/workflow/config/action-grid.tsx | 8 +++++++- lib/db/schema.ts | 8 +++++++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/components/workflow/config/action-config.tsx b/components/workflow/config/action-config.tsx index ccf3d919..7a44915e 100644 --- a/components/workflow/config/action-config.tsx +++ b/components/workflow/config/action-config.tsx @@ -706,15 +706,18 @@ function RunActorFields({
onUpdateConfig("waitForFinish", e.target.checked ? "true" : "false")} - className="h-4 w-4 rounded border-input" + type="checkbox" />
-
- -
- onUpdateConfig("waitForFinish", e.target.checked)} - className="h-4 w-4 rounded border-input" - /> -
- -

- Wait for the Actor to finish and return dataset items -

-
-
); } diff --git a/plugins/apify/steps/run-actor/step.ts b/plugins/apify/steps/run-actor/step.ts index e23a6585..89c404bf 100644 --- a/plugins/apify/steps/run-actor/step.ts +++ b/plugins/apify/steps/run-actor/step.ts @@ -1,10 +1,9 @@ import "server-only"; +import { ApifyClient } from "apify-client"; import { fetchCredentials } from "@/lib/credential-fetcher"; import { getErrorMessage } from "@/lib/utils"; -const APIFY_API_BASE = "https://api.apify.com/v2"; - type ApifyRunActorResult = | { success: true; @@ -16,15 +15,13 @@ type ApifyRunActorResult = | { success: false; error: string }; /** - * Apify Run Actor Step + * Run Apify Actor Step * Runs an Apify Actor and optionally waits for results */ export async function apifyRunActorStep(input: { integrationId?: string; actorId: string; actorInput?: Record; - waitForFinish?: boolean; - maxWaitSecs?: number; }): Promise { "use step"; @@ -42,50 +39,31 @@ export async function apifyRunActorStep(input: { } try { - const waitForFinish = input.waitForFinish !== false; - const maxWaitSecs = input.maxWaitSecs || 120; - - // Start the Actor run - const runUrl = waitForFinish - ? `${APIFY_API_BASE}/acts/${encodeURIComponent(input.actorId)}/run-sync-get-dataset-items?timeout=${maxWaitSecs}` - : `${APIFY_API_BASE}/acts/${encodeURIComponent(input.actorId)}/runs`; + const client = new ApifyClient({ token: apiKey }); + const actorClient = client.actor(input.actorId); + const maxWaitSecs = 120; - const runResponse = await fetch(runUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify(input.actorInput || {}), - }); + // Run synchronously and wait for completion + const runData = await actorClient.call(input.actorInput || {}, { + waitSecs: maxWaitSecs, + }); - if (!runResponse.ok) { - const errorText = await runResponse.text().catch(() => "Unknown error"); - return { - success: false, - error: `Failed to run Actor: ${runResponse.status} - ${errorText}`, - }; - } + // Get dataset items + let data: unknown[] = []; + if (runData.defaultDatasetId) { + const datasetItems = await client + .dataset(runData.defaultDatasetId) + .listItems(); + data = datasetItems.items; + } - if (waitForFinish) { - // For sync runs, we get the dataset items directly - const data = await runResponse.json(); return { - success: true, - runId: runResponse.headers.get("x-apify-run-id") || "unknown", - status: "SUCCEEDED", - data: Array.isArray(data) ? data : [data], + success: true, + runId: runData.id || "unknown", + status: runData.status || "SUCCEEDED", + datasetId: runData.defaultDatasetId, + data, }; - } - - // For async runs, we get the run info - const runData = await runResponse.json(); - return { - success: true, - runId: runData.data?.id || "unknown", - status: runData.data?.status || "RUNNING", - datasetId: runData.data?.defaultDatasetId, - }; } catch (error) { return { success: false, diff --git a/plugins/apify/steps/scrape-single-url/config.tsx b/plugins/apify/steps/scrape-single-url/config.tsx new file mode 100644 index 00000000..e3c0db30 --- /dev/null +++ b/plugins/apify/steps/scrape-single-url/config.tsx @@ -0,0 +1,62 @@ +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { TemplateBadgeInput } from "@/components/ui/template-badge-input"; + +/** + * Scrape Single URL Config Fields Component + * UI for configuring the scrape single URL action + */ +export function ScrapeSingleUrlConfigFields({ + config, + onUpdateConfig, + disabled, +}: { + config: Record; + onUpdateConfig: (key: string, value: unknown) => void; + disabled?: boolean; +}) { + return ( +
+
+ + onUpdateConfig("url", value)} + placeholder="https://example.com or {{NodeName.url}}" + value={(config?.url as string) || ""} + /> +

+ Enter the URL to scrape or use a template reference. +

+
+ +
+ + +

+ Select the crawler type to use for scraping. +

+
+
+ ); +} diff --git a/plugins/apify/steps/scrape-single-url/step.ts b/plugins/apify/steps/scrape-single-url/step.ts new file mode 100644 index 00000000..5adf2945 --- /dev/null +++ b/plugins/apify/steps/scrape-single-url/step.ts @@ -0,0 +1,99 @@ +import "server-only"; + +import { ApifyClient } from "apify-client"; +import { fetchCredentials } from "@/lib/credential-fetcher"; +import { getErrorMessage } from "@/lib/utils"; + +type ScrapeSingleUrlResult = + | { + success: true; + runId: string; + status: string; + markdown?: string; + } + | { success: false; error: string }; + +/** + * Scrape Single URL Step + * Scrapes a single URL using apify/website-content-crawler and returns markdown + */ +export async function scrapeSingleUrlStep(input: { + integrationId?: string; + url: string; + crawlerType?: string; +}): Promise { + "use step"; + + const credentials = input.integrationId + ? await fetchCredentials(input.integrationId) + : {}; + + const apiKey = credentials.APIFY_API_KEY; + + if (!apiKey) { + return { + success: false, + error: "Apify API Token is not configured.", + }; + } + + if (!input.url) { + return { + success: false, + error: "URL is required.", + }; + } + + try { + const client = new ApifyClient({ token: apiKey }); + const actorClient = client.actor("apify/website-content-crawler"); + const crawlerType = input.crawlerType || "playwright"; + + // Prepare actor input + const actorInput = { + startUrls: [{ url: input.url }], + crawlerType, + maxCrawlDepth: 0, + maxCrawlPages: 1, + maxResults: 1, + proxyConfiguration: { + useApifyProxy: true, + }, + removeCookieWarnings: true, + saveHtml: true, + saveMarkdown: true, + }; + + // Run synchronously and wait for completion + const maxWaitSecs = 120; + const runData = await actorClient.call(actorInput, { + waitSecs: maxWaitSecs, + }); + + // Get dataset items + let markdown: string | undefined; + if (runData.defaultDatasetId) { + const datasetItems = await client + .dataset(runData.defaultDatasetId) + .listItems(); + + // Extract markdown from the first item + if (datasetItems.items && datasetItems.items.length > 0) { + const firstItem = datasetItems.items[0] as Record; + markdown = (firstItem.markdown as string) || (firstItem.text as string) || undefined; + } + } + + return { + success: true, + runId: runData.id || "unknown", + status: runData.status || "SUCCEEDED", + markdown, + }; + } catch (error) { + return { + success: false, + error: `Failed to scrape URL: ${getErrorMessage(error)}`, + }; + } +} diff --git a/plugins/apify/test.ts b/plugins/apify/test.ts index 681a516b..8382720f 100644 --- a/plugins/apify/test.ts +++ b/plugins/apify/test.ts @@ -1,18 +1,10 @@ +import { ApifyClient } from "apify-client"; + export async function testApify(credentials: Record) { try { - // Test the API token by fetching user info - const response = await fetch("https://api.apify.com/v2/users/me", { - method: "GET", - headers: { - Authorization: `Bearer ${credentials.APIFY_API_KEY}`, - }, - }); - - if (response.ok) { - return { success: true }; - } - const error = await response.text(); - return { success: false, error }; + const client = new ApifyClient({ token: credentials.APIFY_API_KEY }); + await client.user("me").get(); + return { success: true }; } catch (error) { return { success: false, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1e5a7f42..e503962f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -38,6 +38,9 @@ importers: ai: specifier: ^5.0.102 version: 5.0.102(zod@4.1.12) + apify-client: + specifier: ^2.20.0 + version: 2.20.0 better-auth: specifier: ^1.3.34 version: 1.3.34(next@16.0.1(@babel/core@7.28.5)(@opentelemetry/api@1.9.0)(react-dom@19.2.0(react@19.2.0))(react@19.2.0))(react-dom@19.2.0(react@19.2.0))(react@19.2.0) @@ -167,6 +170,15 @@ packages: resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} + '@apify/consts@2.48.0': + resolution: {integrity: sha512-a0HeYDxAbbkRxc9z2N6beMFAmAJSgBw8WuKUwV+KmCuPyGUVLp54fYzjQ63p9Gv5IVFC88/HMXpAzI29ARgO5w==} + + '@apify/log@2.5.28': + resolution: {integrity: sha512-jU8qIvU+Crek8glBjFl3INjJQWWDR9n2z9Dr0WvUI8KJi0LG9fMdTvV+Aprf9z1b37CbHXgiZkA1iPlNYxKOEQ==} + + '@apify/utilities@2.23.4': + resolution: {integrity: sha512-1tLXOJBJR1SUSp/iEj6kcvV+9B5dn1mvIWDtRYwevJXXURyJdPwzJApi0F0DZz/Vk2HeCC381gnSqASzXN8MLA==} + '@aws-crypto/sha256-browser@5.2.0': resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==} @@ -386,6 +398,10 @@ packages: '@clack/prompts@0.11.0': resolution: {integrity: sha512-pMN5FcrEw9hUkZA4f+zLlzivQSeQf5dRGJjSUbvVYDLvpKCdQx5OaknvKzgbtXOizhP+SJJJjqEbOe55uKKfAw==} + '@crawlee/types@3.15.3': + resolution: {integrity: sha512-RvgVPXrsQw4GQIUXrC1z1aNOedUPJnZ/U/8n+jZ0fu1Iw9moJVMuiuIxSI8q1P6BA84aWZdalyfDWBZ3FMjsiw==} + engines: {node: '>=16.0.0'} + '@drizzle-team/brocli@0.10.2': resolution: {integrity: sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w==} @@ -1864,6 +1880,10 @@ packages: resolution: {integrity: sha512-HcWLW28yTMGXpwE9VLx9J+N2KEUaELadLrkPEEI9tpI5la70xNEVEsu/C+m3u7uoq4FulLqZQhgBCzR9IZhFpA==} engines: {node: '>=20.0.0'} + '@sindresorhus/is@4.6.0': + resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==} + engines: {node: '>=10'} + '@sindresorhus/merge-streams@4.0.0': resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} engines: {node: '>=18'} @@ -2398,6 +2418,10 @@ packages: ansi-align@3.0.1: resolution: {integrity: sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==} + ansi-colors@4.1.3: + resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} + engines: {node: '>=6'} + ansi-escapes@4.3.2: resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==} engines: {node: '>=8'} @@ -2426,6 +2450,9 @@ packages: resolution: {integrity: sha512-0qWUglt9JEqLFr3w1I1pbrChn1grhaiAR2ocX1PP/flRmxgtwTzPFFFnfIlD6aMOLQZgSuCRlidD70lvx8yhzg==} engines: {node: '>=14'} + apify-client@2.20.0: + resolution: {integrity: sha512-oEMTImVVRZ5n8JkFV6dgbBFL3Xqz+GTwjUCjn/hwSNkow31Q8VNGk4qYDfRjkoqNQJ3ZirhtCwTnhkSXn1Tf+g==} + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -2444,6 +2471,9 @@ packages: resolution: {integrity: sha512-UOCGPYbl0tv8+006qks/dTgV9ajs97X2p0FAbyS2iyCRrmLSRolDaHdp+v/CLgnzHc3fVB+CwYiUmei7ndFcgA==} engines: {node: '>=12.0.0'} + async-retry@1.3.3: + resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} + async@3.2.6: resolution: {integrity: sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==} @@ -2618,6 +2648,10 @@ packages: resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} engines: {node: ^14.18.0 || >=16.10.0} + content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} @@ -2736,6 +2770,10 @@ packages: dompurify@3.1.7: resolution: {integrity: sha512-VaTstWtsneJY8xzy7DekmYWEOZcmzIe3Qb3zPd4STve1OBTa+e+WmS1ITQec1fZYXI3HCsOZZiSMpG6oxoWMWQ==} + dot-prop@6.0.1: + resolution: {integrity: sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==} + engines: {node: '>=10'} + dotenv@17.2.3: resolution: {integrity: sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==} engines: {node: '>=12'} @@ -3192,6 +3230,10 @@ packages: resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} engines: {node: '>=0.12.0'} + is-obj@2.0.0: + resolution: {integrity: sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==} + engines: {node: '>=8'} + is-plain-obj@4.1.0: resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} engines: {node: '>=12'} @@ -3394,6 +3436,10 @@ packages: resolution: {integrity: sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + lodash.isequal@4.5.0: + resolution: {integrity: sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==} + deprecated: This package is deprecated. Use require('node:util').isDeepStrictEqual instead. + log-symbols@6.0.0: resolution: {integrity: sha512-i24m8rpwhmPIS4zscNzK6MSEhk0DUWa/8iYQWxhffV8jkI4Phvs3F+quL5xvS0gdQR0FyTCMMH33Y78dDTzzIw==} engines: {node: '>=18'} @@ -3615,6 +3661,10 @@ packages: resolution: {integrity: sha512-wrAwOeXp1RRMFfQY8Sy7VaGVmPocaLwSFOYCGKSyo8qmJ+/yaafCl5BCA1IQZWqFSRBrKDYFeR9d/VyQzfH/jg==} engines: {node: '>= 6.0'} + ow@0.28.2: + resolution: {integrity: sha512-dD4UpyBh/9m4X2NVjA+73/ZPBRF+uF4zIMFvvQsabMiEK8x41L3rQ8EENOi35kyyoaJwNxEeJcP6Fj1H4U409Q==} + engines: {node: '>=12'} + oxc-resolver@11.13.2: resolution: {integrity: sha512-1SXVyYQ9bqMX3uZo8Px81EG7jhZkO9PvvR5X9roY5TLYVm4ZA7pbPDNlYaDBBeF9U+YO3OeMNoHde52hrcCu8w==} @@ -4179,6 +4229,10 @@ packages: resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==} hasBin: true + vali-date@1.0.0: + resolution: {integrity: sha512-sgECfZthyaCKW10N0fm27cg8HYTFK5qMWgypqkXMQ4Wbl/zZKx7xZICgcoxIIE+WFAP/MBL2EFwC/YvLxw3Zeg==} + engines: {node: '>=0.10.0'} + vaul@1.1.2: resolution: {integrity: sha512-ZFkClGpWyI2WUQjdLJ/BaGuV6AVQiJ3uELGk3OYtP+B6yCO7Cmn9vPFXVJkRaGkOJu3m8bQMgtyzNHixULceQA==} peerDependencies: @@ -4328,6 +4382,18 @@ snapshots: '@alloc/quick-lru@5.2.0': {} + '@apify/consts@2.48.0': {} + + '@apify/log@2.5.28': + dependencies: + '@apify/consts': 2.48.0 + ansi-colors: 4.1.3 + + '@apify/utilities@2.23.4': + dependencies: + '@apify/consts': 2.48.0 + '@apify/log': 2.5.28 + '@aws-crypto/sha256-browser@5.2.0': dependencies: '@aws-crypto/sha256-js': 5.2.0 @@ -4858,6 +4924,10 @@ snapshots: picocolors: 1.1.1 sisteransi: 1.0.5 + '@crawlee/types@3.15.3': + dependencies: + tslib: 2.8.1 + '@drizzle-team/brocli@0.10.2': {} '@emnapi/core@1.7.1': @@ -6256,6 +6326,8 @@ snapshots: '@peculiar/asn1-x509': 2.5.0 '@peculiar/x509': 1.14.0 + '@sindresorhus/is@4.6.0': {} + '@sindresorhus/merge-streams@4.0.0': {} '@slack/logger@4.0.0': @@ -6996,6 +7068,8 @@ snapshots: dependencies: string-width: 4.2.3 + ansi-colors@4.1.3: {} + ansi-escapes@4.3.2: dependencies: type-fest: 0.21.3 @@ -7016,6 +7090,22 @@ snapshots: ansis@3.17.0: {} + apify-client@2.20.0: + dependencies: + '@apify/consts': 2.48.0 + '@apify/log': 2.5.28 + '@apify/utilities': 2.23.4 + '@crawlee/types': 3.15.3 + ansi-colors: 4.1.3 + async-retry: 1.3.3 + axios: 1.13.1 + content-type: 1.0.5 + ow: 0.28.2 + tslib: 2.8.1 + type-fest: 4.41.0 + transitivePeerDependencies: + - debug + argparse@2.0.1: {} aria-hidden@1.2.6: @@ -7032,6 +7122,10 @@ snapshots: pvutils: 1.1.5 tslib: 2.8.1 + async-retry@1.3.3: + dependencies: + retry: 0.13.1 + async@3.2.6: {} asynckit@0.4.0: {} @@ -7203,6 +7297,8 @@ snapshots: consola@3.4.2: {} + content-type@1.0.5: {} + convert-source-map@2.0.0: optional: true @@ -7303,6 +7399,10 @@ snapshots: dompurify@3.1.7: {} + dot-prop@6.0.1: + dependencies: + is-obj: 2.0.0 + dotenv@17.2.3: {} drizzle-kit@0.31.6: @@ -7693,6 +7793,8 @@ snapshots: is-number@7.0.0: {} + is-obj@2.0.0: {} + is-plain-obj@4.1.0: {} is-stream@2.0.1: {} @@ -7856,6 +7958,8 @@ snapshots: dependencies: p-locate: 6.0.0 + lodash.isequal@4.5.0: {} + log-symbols@6.0.0: dependencies: chalk: 5.6.2 @@ -8056,6 +8160,14 @@ snapshots: os-paths@4.4.0: {} + ow@0.28.2: + dependencies: + '@sindresorhus/is': 4.6.0 + callsites: 3.1.0 + dot-prop: 6.0.1 + lodash.isequal: 4.5.0 + vali-date: 1.0.0 + oxc-resolver@11.13.2: optionalDependencies: '@oxc-resolver/binding-android-arm-eabi': 11.13.2 @@ -8632,6 +8744,8 @@ snapshots: uuid@10.0.0: {} + vali-date@1.0.0: {} + vaul@1.1.2(@types/react-dom@19.2.2(@types/react@19.2.2))(@types/react@19.2.2)(react-dom@19.2.0(react@19.2.0))(react@19.2.0): dependencies: '@radix-ui/react-dialog': 1.1.15(@types/react-dom@19.2.2(@types/react@19.2.2))(@types/react@19.2.2)(react-dom@19.2.0(react@19.2.0))(react@19.2.0) diff --git a/public/integrations/apify.svg b/public/integrations/apify.svg new file mode 100644 index 00000000..c8894c84 --- /dev/null +++ b/public/integrations/apify.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + From 93aee74905191b152261c37341bcb00c9e707014 Mon Sep 17 00:00:00 2001 From: drobnikj Date: Fri, 28 Nov 2025 15:46:04 +0100 Subject: [PATCH 06/18] fix: adding apify-client package --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index a424e88a..603cd3a1 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "@vercel/sdk": "^1.17.1", "@xyflow/react": "^12.9.2", "ai": "^5.0.102", + "apify-client": "^2.20.0", "better-auth": "^1.3.34", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", From ed8b30ce3309bf97965c1a2e472f3570cf3dd94c Mon Sep 17 00:00:00 2001 From: drobnikj Date: Fri, 28 Nov 2025 15:59:32 +0100 Subject: [PATCH 07/18] fix: update action --- components/workflow/config/action-config.tsx | 19 +++---------- plugins/apify/steps/run-actor/config.tsx | 28 +++++--------------- 2 files changed, 10 insertions(+), 37 deletions(-) diff --git a/components/workflow/config/action-config.tsx b/components/workflow/config/action-config.tsx index 4bbadac0..b27445b8 100644 --- a/components/workflow/config/action-config.tsx +++ b/components/workflow/config/action-config.tsx @@ -664,7 +664,7 @@ function RunActorFields({ return ( <>
- + { - try { - JSON.parse(value); - onUpdateConfig("actorInput", value); - } catch { - // Store as string if not valid JSON yet (user is still typing) - onUpdateConfig("actorInputRaw", value); - } - }} placeholder='{"startUrls": [{"url": "https://example.com"}]}' rows={6} - value={ - (config?.actorInputRaw as string) || - (config?.actorInput - ? JSON.stringify(config.actorInput, null, 2) - : "") - } + onChange={(value) => onUpdateConfig("actorInput", value)} + value={(config?.actorInput as string) || ""} />

JSON input for the Actor. Check the Actor's documentation for required diff --git a/plugins/apify/steps/run-actor/config.tsx b/plugins/apify/steps/run-actor/config.tsx index 75d6d451..1ed183e2 100644 --- a/plugins/apify/steps/run-actor/config.tsx +++ b/plugins/apify/steps/run-actor/config.tsx @@ -18,7 +18,7 @@ export function RunActorConfigFields({ return (

- + { - try { - const parsed = JSON.parse(value); - onUpdateConfig("actorInput", parsed); - onUpdateConfig("actorInputRaw", value); - } catch { - // Store as string if not valid JSON yet (user is still typing) - onUpdateConfig("actorInputRaw", value); - } - }} - placeholder='{"startUrls": [{"url": "https://example.com"}]}' - rows={6} - value={ - (config?.actorInputRaw as string) || - (config?.actorInput - ? JSON.stringify(config.actorInput, null, 2) - : "") - } + disabled={disabled} + id="actorInput" + placeholder='{"startUrls": [{"url": "https://example.com"}]}' + rows={6} + onChange={(value) => onUpdateConfig("actorInput", value)} + value={(config?.actorInput as string) || ""} />

JSON input for the Actor. Check the Actor's documentation for required From 61207bc8442a3960d825821d2ec92ac164934654 Mon Sep 17 00:00:00 2001 From: drobnikj Date: Mon, 1 Dec 2025 11:52:59 +0100 Subject: [PATCH 08/18] fix: after merge fixes --- README.md | 1 + .../settings/integration-form-dialog.tsx | 3 +- components/settings/integrations-manager.tsx | 3 +- components/ui/drawer.tsx | 8 +- components/ui/dropdown-menu.tsx | 8 +- components/ui/tabs.tsx | 1 + .../config/action-config-renderer.tsx | 6 + components/workflow/nodes/action-node.tsx | 1 - lib/credential-fetcher.ts | 1 - lib/step-registry.ts | 20 ++- lib/types/integration.ts | 3 +- package.json | 3 +- plugins/apify/index.tsx | 66 ++++++++-- pnpm-lock.yaml | 114 ++++++++++++++++++ 14 files changed, 212 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index b08be906..c8d580e2 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ Visit [http://localhost:3000](http://localhost:3000) to get started. - **AI Gateway**: Generate Text, Generate Image +- **Apify**: Run Apify Actor, Scrape Single URL - **Firecrawl**: Scrape URL, Search Web - **Linear**: Create Ticket, Find Issues - **Resend**: Send Email diff --git a/components/settings/integration-form-dialog.tsx b/components/settings/integration-form-dialog.tsx index edf267fd..9312cc53 100644 --- a/components/settings/integration-form-dialog.tsx +++ b/components/settings/integration-form-dialog.tsx @@ -187,9 +187,8 @@ export function IntegrationFormDialog({

)); }; -} -return ( + return ( !isOpen && onClose()} open={open}> diff --git a/components/settings/integrations-manager.tsx b/components/settings/integrations-manager.tsx index e36428e1..4f72fb3c 100644 --- a/components/settings/integrations-manager.tsx +++ b/components/settings/integrations-manager.tsx @@ -1,6 +1,7 @@ "use client"; import { Pencil, Trash2 } from "lucide-react"; +import { useCallback, useEffect, useState } from "react"; import { toast } from "sonner"; import { AlertDialog, @@ -15,7 +16,7 @@ import { import { Button } from "@/components/ui/button"; import { IntegrationIcon } from "@/components/ui/integration-icon"; import { Spinner } from "@/components/ui/spinner"; -import { api } from "@/lib/api-client"; +import { api, type Integration } from "@/lib/api-client"; import { getIntegrationLabels } from "@/plugins"; import { IntegrationFormDialog } from "./integration-form-dialog"; diff --git a/components/ui/drawer.tsx b/components/ui/drawer.tsx index 8aa69230..d21227cc 100644 --- a/components/ui/drawer.tsx +++ b/components/ui/drawer.tsx @@ -8,7 +8,13 @@ import { cn } from "@/lib/utils" function Drawer({ ...props }: React.ComponentProps) { - return + return ( + + ); } function DrawerTrigger({ diff --git a/components/ui/dropdown-menu.tsx b/components/ui/dropdown-menu.tsx index 4246c7d9..d615bfc6 100644 --- a/components/ui/dropdown-menu.tsx +++ b/components/ui/dropdown-menu.tsx @@ -9,7 +9,13 @@ import { cn } from "@/lib/utils" function DropdownMenu({ ...props }: React.ComponentProps) { - return + return ( + + ); } function DropdownMenuPortal({ diff --git a/components/ui/tabs.tsx b/components/ui/tabs.tsx index cf2d1907..0f6ea6cd 100644 --- a/components/ui/tabs.tsx +++ b/components/ui/tabs.tsx @@ -13,6 +13,7 @@ function Tabs({ ); diff --git a/components/workflow/config/action-config-renderer.tsx b/components/workflow/config/action-config-renderer.tsx index 030c20ba..e155abbc 100644 --- a/components/workflow/config/action-config-renderer.tsx +++ b/components/workflow/config/action-config-renderer.tsx @@ -224,6 +224,12 @@ export function ActionConfigRenderer({ onUpdateConfig, disabled, }: ActionConfigRendererProps) { + // Safety check: ensure fields is an array + if (!Array.isArray(fields)) { + console.warn("ActionConfigRenderer: fields must be an array", fields); + return null; + } + return ( <> {fields.map((field) => { diff --git a/components/workflow/nodes/action-node.tsx b/components/workflow/nodes/action-node.tsx index 3a788812..c5cdf51d 100644 --- a/components/workflow/nodes/action-node.tsx +++ b/components/workflow/nodes/action-node.tsx @@ -89,7 +89,6 @@ const getIntegrationFromActionType = (actionType: string): string => { } return "System"; - >>>>>>> origin/main }; // Helper to detect if output is a base64 image from generateImage step diff --git a/lib/credential-fetcher.ts b/lib/credential-fetcher.ts index 6167b177..430acb85 100644 --- a/lib/credential-fetcher.ts +++ b/lib/credential-fetcher.ts @@ -35,7 +35,6 @@ const SYSTEM_CREDENTIAL_MAPPERS: Record< }, }; ->>>>>>> origin/main /** * Map integration config to WorkflowCredentials format * Uses plugin registry for plugin integrations, hardcoded mappers for system integrations diff --git a/lib/step-registry.ts b/lib/step-registry.ts index 6a71649b..50558723 100644 --- a/lib/step-registry.ts +++ b/lib/step-registry.ts @@ -7,7 +7,7 @@ * This registry enables dynamic step imports that are statically analyzable * by the bundler. Each action type maps to its step importer function. * - * Generated entries: 10 + * Generated entries: 12 */ import "server-only"; @@ -41,11 +41,19 @@ export const PLUGIN_STEP_IMPORTERS: Record = { importer: () => import("@/plugins/ai-gateway/steps/generate-image"), stepFunction: "generateImageStep", }, + "apify/run-actor": { + importer: () => import("@/plugins/apify/steps/run-actor/step"), + stepFunction: "apifyRunActorStep", + }, + "apify/scrape-single-url": { + importer: () => import("@/plugins/apify/steps/scrape-single-url/step"), + stepFunction: "scrapeSingleUrlStep", + }, "firecrawl/scrape": { importer: () => import("@/plugins/firecrawl/steps/scrape"), stepFunction: "firecrawlScrapeStep", }, - Scrape: { + "Scrape": { importer: () => import("@/plugins/firecrawl/steps/scrape"), stepFunction: "firecrawlScrapeStep", }, @@ -53,7 +61,7 @@ export const PLUGIN_STEP_IMPORTERS: Record = { importer: () => import("@/plugins/firecrawl/steps/search"), stepFunction: "firecrawlSearchStep", }, - Search: { + "Search": { importer: () => import("@/plugins/firecrawl/steps/search"), stepFunction: "firecrawlSearchStep", }, @@ -114,6 +122,8 @@ export const PLUGIN_STEP_IMPORTERS: Record = { export const ACTION_LABELS: Record = { "ai-gateway/generate-text": "Generate Text", "ai-gateway/generate-image": "Generate Image", + "apify/run-actor": "Run Apify Actor", + "apify/scrape-single-url": "Scrape Single URL", "firecrawl/scrape": "Scrape URL", "firecrawl/search": "Search Web", "linear/create-ticket": "Create Ticket", @@ -122,8 +132,8 @@ export const ACTION_LABELS: Record = { "slack/send-message": "Send Slack Message", "v0/create-chat": "Create Chat", "v0/send-message": "Send Message", - Scrape: "Scrape URL", - Search: "Search Web", + "Scrape": "Scrape URL", + "Search": "Search Web", "Generate Text": "Generate Text", "Generate Image": "Generate Image", "Send Email": "Send Email", diff --git a/lib/types/integration.ts b/lib/types/integration.ts index cd146253..551ce9b2 100644 --- a/lib/types/integration.ts +++ b/lib/types/integration.ts @@ -9,12 +9,13 @@ * 2. Add a system integration to SYSTEM_INTEGRATION_TYPES in discover-plugins.ts * 3. Run: pnpm discover-plugins * - * Generated types: ai-gateway, database, firecrawl, linear, resend, slack, v0 + * Generated types: ai-gateway, apify, database, firecrawl, linear, resend, slack, v0 */ // Integration type union - plugins + system integrations export type IntegrationType = | "ai-gateway" + | "apify" | "database" | "firecrawl" | "linear" diff --git a/package.json b/package.json index 94ebdeff..6bfb35c7 100644 --- a/package.json +++ b/package.json @@ -72,5 +72,6 @@ "tw-animate-css": "^1.4.0", "typescript": "^5", "ultracite": "6.3.0" - } + }, + "packageManager": "pnpm@10.6.1+sha512.40ee09af407fa9fbb5fbfb8e1cb40fbb74c0af0c3e10e9224d7b53c7658528615b2c92450e74cfad91e3a2dcafe3ce4050d80bda71d757756d2ce2b66213e9a3" } diff --git a/plugins/apify/index.tsx b/plugins/apify/index.tsx index 8f40b3e5..7152bb54 100644 --- a/plugins/apify/index.tsx +++ b/plugins/apify/index.tsx @@ -5,8 +5,6 @@ import { runActorCodegenTemplate } from "./codegen/run-actor"; import { scrapeSingleUrlCodegenTemplate } from "./codegen/scrape-single-url"; import { ApifyIcon } from "./icon"; import { ApifySettings } from "./settings"; -import { RunActorConfigFields } from "./steps/run-actor/config"; -import { ScrapeSingleUrlConfigFields } from "./steps/scrape-single-url/config"; import { testApify } from "./test"; const apifyPlugin: IntegrationPlugin = { @@ -14,10 +12,7 @@ const apifyPlugin: IntegrationPlugin = { label: "Apify", description: "Run web scraping and automation Actors", - icon: { - type: "image", - value: "/integrations/apify.svg", - }, + icon: ApifyIcon, settingsComponent: ApifySettings, @@ -50,25 +45,72 @@ const apifyPlugin: IntegrationPlugin = { actions: [ { - id: "Run Apify Actor", + slug: "run-actor", label: "Run Apify Actor", description: "Run an Apify Actor and get results", category: "Apify", icon: ApifyIcon, stepFunction: "apifyRunActorStep", - stepImportPath: "run-actor", - configFields: RunActorConfigFields, + stepImportPath: "run-actor/step", + configFields: [ + { + key: "actorId", + label: "Actor (ID or name)", + type: "template-input", + placeholder: "apify/web-scraper or {{NodeName.actorId}}", + example: "apify/web-scraper", + required: true, + }, + { + key: "actorInput", + label: "Actor Input (JSON)", + type: "template-textarea", + placeholder: '{"startUrls": [{"url": "https://example.com"}]}', + rows: 6, + example: '{"startUrls": [{"url": "https://example.com"}]}', + required: true, + }, + ], codegenTemplate: runActorCodegenTemplate, }, { - id: "Scrape Single URL", + slug: "scrape-single-url", label: "Scrape Single URL", description: "Scrape a single URL and get markdown output", category: "Apify", icon: Globe, stepFunction: "scrapeSingleUrlStep", - stepImportPath: "scrape-single-url", - configFields: ScrapeSingleUrlConfigFields, + stepImportPath: "scrape-single-url/step", + configFields: [ + { + key: "url", + label: "URL", + type: "template-input", + placeholder: "https://example.com or {{NodeName.url}}", + example: "https://example.com", + required: true, + }, + { + key: "crawlerType", + label: "Crawler Type", + type: "select", + defaultValue: "playwright", + options: [ + { + value: "playwright:adaptive", + label: "Adaptive switching between browser and raw HTTP", + }, + { + value: "playwright:firefox", + label: "Headless browser (Firefox+Playwright)", + }, + { + value: "cheerio", + label: "Raw HTTP client (Cheerio)", + }, + ], + }, + ], codegenTemplate: scrapeSingleUrlCodegenTemplate, }, ], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 01d512da..714149b8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44,6 +44,9 @@ importers: ai: specifier: ^5.0.102 version: 5.0.102(zod@4.1.12) + apify-client: + specifier: ^2.20.0 + version: 2.20.0 better-auth: specifier: ^1.3.34 version: 1.3.34(next@16.0.1(@babel/core@7.28.5)(@opentelemetry/api@1.9.0)(react-dom@19.2.0(react@19.2.0))(react@19.2.0))(react-dom@19.2.0(react@19.2.0))(react@19.2.0) @@ -182,6 +185,15 @@ packages: resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} + '@apify/consts@2.48.0': + resolution: {integrity: sha512-a0HeYDxAbbkRxc9z2N6beMFAmAJSgBw8WuKUwV+KmCuPyGUVLp54fYzjQ63p9Gv5IVFC88/HMXpAzI29ARgO5w==} + + '@apify/log@2.5.28': + resolution: {integrity: sha512-jU8qIvU+Crek8glBjFl3INjJQWWDR9n2z9Dr0WvUI8KJi0LG9fMdTvV+Aprf9z1b37CbHXgiZkA1iPlNYxKOEQ==} + + '@apify/utilities@2.23.4': + resolution: {integrity: sha512-1tLXOJBJR1SUSp/iEj6kcvV+9B5dn1mvIWDtRYwevJXXURyJdPwzJApi0F0DZz/Vk2HeCC381gnSqASzXN8MLA==} + '@aws-crypto/sha256-browser@5.2.0': resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==} @@ -401,6 +413,10 @@ packages: '@clack/prompts@0.11.0': resolution: {integrity: sha512-pMN5FcrEw9hUkZA4f+zLlzivQSeQf5dRGJjSUbvVYDLvpKCdQx5OaknvKzgbtXOizhP+SJJJjqEbOe55uKKfAw==} + '@crawlee/types@3.15.3': + resolution: {integrity: sha512-RvgVPXrsQw4GQIUXrC1z1aNOedUPJnZ/U/8n+jZ0fu1Iw9moJVMuiuIxSI8q1P6BA84aWZdalyfDWBZ3FMjsiw==} + engines: {node: '>=16.0.0'} + '@drizzle-team/brocli@0.10.2': resolution: {integrity: sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w==} @@ -2013,6 +2029,10 @@ packages: resolution: {integrity: sha512-HcWLW28yTMGXpwE9VLx9J+N2KEUaELadLrkPEEI9tpI5la70xNEVEsu/C+m3u7uoq4FulLqZQhgBCzR9IZhFpA==} engines: {node: '>=20.0.0'} + '@sindresorhus/is@4.6.0': + resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==} + engines: {node: '>=10'} + '@sindresorhus/merge-streams@4.0.0': resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} engines: {node: '>=18'} @@ -2596,6 +2616,10 @@ packages: ansi-align@3.0.1: resolution: {integrity: sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==} + ansi-colors@4.1.3: + resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} + engines: {node: '>=6'} + ansi-escapes@4.3.2: resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==} engines: {node: '>=8'} @@ -2624,6 +2648,9 @@ packages: resolution: {integrity: sha512-0qWUglt9JEqLFr3w1I1pbrChn1grhaiAR2ocX1PP/flRmxgtwTzPFFFnfIlD6aMOLQZgSuCRlidD70lvx8yhzg==} engines: {node: '>=14'} + apify-client@2.20.0: + resolution: {integrity: sha512-oEMTImVVRZ5n8JkFV6dgbBFL3Xqz+GTwjUCjn/hwSNkow31Q8VNGk4qYDfRjkoqNQJ3ZirhtCwTnhkSXn1Tf+g==} + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -2642,6 +2669,9 @@ packages: resolution: {integrity: sha512-UOCGPYbl0tv8+006qks/dTgV9ajs97X2p0FAbyS2iyCRrmLSRolDaHdp+v/CLgnzHc3fVB+CwYiUmei7ndFcgA==} engines: {node: '>=12.0.0'} + async-retry@1.3.3: + resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} + async@3.2.6: resolution: {integrity: sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==} @@ -2823,6 +2853,10 @@ packages: resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} engines: {node: ^14.18.0 || >=16.10.0} + content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} @@ -2941,6 +2975,10 @@ packages: dompurify@3.1.7: resolution: {integrity: sha512-VaTstWtsneJY8xzy7DekmYWEOZcmzIe3Qb3zPd4STve1OBTa+e+WmS1ITQec1fZYXI3HCsOZZiSMpG6oxoWMWQ==} + dot-prop@6.0.1: + resolution: {integrity: sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==} + engines: {node: '>=10'} + dotenv@17.2.3: resolution: {integrity: sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==} engines: {node: '>=12'} @@ -3406,6 +3444,10 @@ packages: resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} engines: {node: '>=0.12.0'} + is-obj@2.0.0: + resolution: {integrity: sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==} + engines: {node: '>=8'} + is-plain-obj@4.1.0: resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} engines: {node: '>=12'} @@ -3608,6 +3650,10 @@ packages: resolution: {integrity: sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + lodash.isequal@4.5.0: + resolution: {integrity: sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==} + deprecated: This package is deprecated. Use require('node:util').isDeepStrictEqual instead. + log-symbols@6.0.0: resolution: {integrity: sha512-i24m8rpwhmPIS4zscNzK6MSEhk0DUWa/8iYQWxhffV8jkI4Phvs3F+quL5xvS0gdQR0FyTCMMH33Y78dDTzzIw==} engines: {node: '>=18'} @@ -3833,6 +3879,10 @@ packages: resolution: {integrity: sha512-wrAwOeXp1RRMFfQY8Sy7VaGVmPocaLwSFOYCGKSyo8qmJ+/yaafCl5BCA1IQZWqFSRBrKDYFeR9d/VyQzfH/jg==} engines: {node: '>= 6.0'} + ow@0.28.2: + resolution: {integrity: sha512-dD4UpyBh/9m4X2NVjA+73/ZPBRF+uF4zIMFvvQsabMiEK8x41L3rQ8EENOi35kyyoaJwNxEeJcP6Fj1H4U409Q==} + engines: {node: '>=12'} + oxc-resolver@11.13.2: resolution: {integrity: sha512-1SXVyYQ9bqMX3uZo8Px81EG7jhZkO9PvvR5X9roY5TLYVm4ZA7pbPDNlYaDBBeF9U+YO3OeMNoHde52hrcCu8w==} @@ -4409,6 +4459,10 @@ packages: resolution: {integrity: sha512-4MYBu2UuYq6wwNtqlOTUobeUYjXH+RzpSFRQKWOlRw18T47mjGq5Tp4odGS0GK7OGnUwxKG2Cm6JkLx6RLWmBA==} engines: {node: '>=22', pnpm: '>=9'} + vali-date@1.0.0: + resolution: {integrity: sha512-sgECfZthyaCKW10N0fm27cg8HYTFK5qMWgypqkXMQ4Wbl/zZKx7xZICgcoxIIE+WFAP/MBL2EFwC/YvLxw3Zeg==} + engines: {node: '>=0.10.0'} + vaul@1.1.2: resolution: {integrity: sha512-ZFkClGpWyI2WUQjdLJ/BaGuV6AVQiJ3uELGk3OYtP+B6yCO7Cmn9vPFXVJkRaGkOJu3m8bQMgtyzNHixULceQA==} peerDependencies: @@ -4558,6 +4612,18 @@ snapshots: '@alloc/quick-lru@5.2.0': {} + '@apify/consts@2.48.0': {} + + '@apify/log@2.5.28': + dependencies: + '@apify/consts': 2.48.0 + ansi-colors: 4.1.3 + + '@apify/utilities@2.23.4': + dependencies: + '@apify/consts': 2.48.0 + '@apify/log': 2.5.28 + '@aws-crypto/sha256-browser@5.2.0': dependencies: '@aws-crypto/sha256-js': 5.2.0 @@ -5088,6 +5154,10 @@ snapshots: picocolors: 1.1.1 sisteransi: 1.0.5 + '@crawlee/types@3.15.3': + dependencies: + tslib: 2.8.1 + '@drizzle-team/brocli@0.10.2': {} '@emnapi/core@1.7.1': @@ -6605,6 +6675,8 @@ snapshots: '@peculiar/asn1-x509': 2.5.0 '@peculiar/x509': 1.14.0 + '@sindresorhus/is@4.6.0': {} + '@sindresorhus/merge-streams@4.0.0': {} '@slack/logger@4.0.0': @@ -7355,6 +7427,8 @@ snapshots: dependencies: string-width: 4.2.3 + ansi-colors@4.1.3: {} + ansi-escapes@4.3.2: dependencies: type-fest: 0.21.3 @@ -7375,6 +7449,22 @@ snapshots: ansis@3.17.0: {} + apify-client@2.20.0: + dependencies: + '@apify/consts': 2.48.0 + '@apify/log': 2.5.28 + '@apify/utilities': 2.23.4 + '@crawlee/types': 3.15.3 + ansi-colors: 4.1.3 + async-retry: 1.3.3 + axios: 1.13.1 + content-type: 1.0.5 + ow: 0.28.2 + tslib: 2.8.1 + type-fest: 4.41.0 + transitivePeerDependencies: + - debug + argparse@2.0.1: {} aria-hidden@1.2.6: @@ -7391,6 +7481,10 @@ snapshots: pvutils: 1.1.5 tslib: 2.8.1 + async-retry@1.3.3: + dependencies: + retry: 0.13.1 + async@3.2.6: {} asynckit@0.4.0: {} @@ -7566,6 +7660,8 @@ snapshots: consola@3.4.2: {} + content-type@1.0.5: {} + convert-source-map@2.0.0: optional: true @@ -7666,6 +7762,10 @@ snapshots: dompurify@3.1.7: {} + dot-prop@6.0.1: + dependencies: + is-obj: 2.0.0 + dotenv@17.2.3: {} drizzle-kit@0.31.6: @@ -8063,6 +8163,8 @@ snapshots: is-number@7.0.0: {} + is-obj@2.0.0: {} + is-plain-obj@4.1.0: {} is-stream@2.0.1: {} @@ -8226,6 +8328,8 @@ snapshots: dependencies: p-locate: 6.0.0 + lodash.isequal@4.5.0: {} + log-symbols@6.0.0: dependencies: chalk: 5.6.2 @@ -8428,6 +8532,14 @@ snapshots: os-paths@4.4.0: {} + ow@0.28.2: + dependencies: + '@sindresorhus/is': 4.6.0 + callsites: 3.1.0 + dot-prop: 6.0.1 + lodash.isequal: 4.5.0 + vali-date: 1.0.0 + oxc-resolver@11.13.2: optionalDependencies: '@oxc-resolver/binding-android-arm-eabi': 11.13.2 @@ -9015,6 +9127,8 @@ snapshots: v0-sdk@0.15.1: {} + vali-date@1.0.0: {} + vaul@1.1.2(@types/react-dom@19.2.2(@types/react@19.2.2))(@types/react@19.2.2)(react-dom@19.2.0(react@19.2.0))(react@19.2.0): dependencies: '@radix-ui/react-dialog': 1.1.15(@types/react-dom@19.2.2(@types/react@19.2.2))(@types/react@19.2.2)(react-dom@19.2.0(react@19.2.0))(react@19.2.0) From 850a060a14d045530aa61713903372e59d4d49be Mon Sep 17 00:00:00 2001 From: drobnikj Date: Mon, 1 Dec 2025 12:02:13 +0100 Subject: [PATCH 09/18] fix: sync codegen --- lib/step-registry.ts | 8 +- plugins/apify/codegen/run-actor.ts | 17 ++++- plugins/apify/codegen/scrape-single-url.ts | 87 +++++++++++++--------- 3 files changed, 69 insertions(+), 43 deletions(-) diff --git a/lib/step-registry.ts b/lib/step-registry.ts index 50558723..6e51f4a2 100644 --- a/lib/step-registry.ts +++ b/lib/step-registry.ts @@ -53,7 +53,7 @@ export const PLUGIN_STEP_IMPORTERS: Record = { importer: () => import("@/plugins/firecrawl/steps/scrape"), stepFunction: "firecrawlScrapeStep", }, - "Scrape": { + Scrape: { importer: () => import("@/plugins/firecrawl/steps/scrape"), stepFunction: "firecrawlScrapeStep", }, @@ -61,7 +61,7 @@ export const PLUGIN_STEP_IMPORTERS: Record = { importer: () => import("@/plugins/firecrawl/steps/search"), stepFunction: "firecrawlSearchStep", }, - "Search": { + Search: { importer: () => import("@/plugins/firecrawl/steps/search"), stepFunction: "firecrawlSearchStep", }, @@ -132,8 +132,8 @@ export const ACTION_LABELS: Record = { "slack/send-message": "Send Slack Message", "v0/create-chat": "Create Chat", "v0/send-message": "Send Message", - "Scrape": "Scrape URL", - "Search": "Search Web", + Scrape: "Scrape URL", + Search: "Search Web", "Generate Text": "Generate Text", "Generate Image": "Generate Image", "Send Email": "Send Email", diff --git a/plugins/apify/codegen/run-actor.ts b/plugins/apify/codegen/run-actor.ts index 206078e6..6d953db9 100644 --- a/plugins/apify/codegen/run-actor.ts +++ b/plugins/apify/codegen/run-actor.ts @@ -11,10 +11,16 @@ export async function apifyRunActorStep(input: { }) { "use step"; - const apiKey = process.env.APIFY_API_KEY!; - const client = new ApifyClient({ token: apiKey }); - const actorClient = client.actor(input.actorId); - const maxWaitSecs = 120; + const apiKey = process.env.APIFY_API_KEY; + + if (!apiKey) { + throw new Error("Apify API Token is not configured. Set APIFY_API_KEY environment variable."); + } + + try { + const client = new ApifyClient({ token: apiKey }); + const actorClient = client.actor(input.actorId); + const maxWaitSecs = 120; // Run synchronously and wait for completion const runData = await actorClient.call(input.actorInput || {}, { @@ -36,4 +42,7 @@ export async function apifyRunActorStep(input: { datasetId: runData.defaultDatasetId, data, }; + } catch (error) { + throw new Error(\`Failed to run Actor: \${error instanceof Error ? error.message : String(error)}\`); + } }`; diff --git a/plugins/apify/codegen/scrape-single-url.ts b/plugins/apify/codegen/scrape-single-url.ts index 82eca00d..2b1b026a 100644 --- a/plugins/apify/codegen/scrape-single-url.ts +++ b/plugins/apify/codegen/scrape-single-url.ts @@ -11,45 +11,62 @@ export async function scrapeSingleUrlStep(input: { }) { "use step"; - const apiKey = process.env.APIFY_API_KEY!; - const client = new ApifyClient({ token: apiKey }); - const actorClient = client.actor("apify/website-content-crawler"); - const maxWaitSecs = 120; - const crawlerType = input.crawlerType || "playwright"; + const apiKey = process.env.APIFY_API_KEY; + + if (!apiKey) { + throw new Error("Apify API Token is not configured. Set APIFY_API_KEY environment variable."); + } if (!input.url) { - throw new Error("URL is required"); + throw new Error("URL is required."); } - // Prepare actor input - const actorInput = { - startUrls: [{ url: input.url }], - crawlerType, - outputFormat: "markdown", - }; - - // Run synchronously and wait for completion - const runData = await actorClient.call(actorInput, { - waitSecs: maxWaitSecs, - }); - - // Get dataset items - let markdown: string | undefined; - if (runData.defaultDatasetId) { - const datasetItems = await client - .dataset(runData.defaultDatasetId) - .listItems(); - - // Extract markdown from the first item - if (datasetItems.items && datasetItems.items.length > 0) { - const firstItem = datasetItems.items[0] as Record; - markdown = firstItem.markdown as string || firstItem.text as string || JSON.stringify(firstItem); + try { + const client = new ApifyClient({ token: apiKey }); + const actorClient = client.actor("apify/website-content-crawler"); + const crawlerType = input.crawlerType || "playwright"; + + // Prepare actor input + const actorInput = { + startUrls: [{ url: input.url }], + crawlerType, + maxCrawlDepth: 0, + maxCrawlPages: 1, + maxResults: 1, + proxyConfiguration: { + useApifyProxy: true, + }, + removeCookieWarnings: true, + saveHtml: true, + saveMarkdown: true, + }; + + // Run synchronously and wait for completion + const maxWaitSecs = 120; + const runData = await actorClient.call(actorInput, { + waitSecs: maxWaitSecs, + }); + + // Get dataset items + let markdown: string | undefined; + if (runData.defaultDatasetId) { + const datasetItems = await client + .dataset(runData.defaultDatasetId) + .listItems(); + + // Extract markdown from the first item + if (datasetItems.items && datasetItems.items.length > 0) { + const firstItem = datasetItems.items[0] as Record; + markdown = (firstItem.markdown as string) || (firstItem.text as string) || undefined; + } } - } - return { - runId: runData.id || "unknown", - status: runData.status || "SUCCEEDED", - markdown, - }; + return { + runId: runData.id || "unknown", + status: runData.status || "SUCCEEDED", + markdown, + }; + } catch (error) { + throw new Error(\`Failed to scrape URL: \${error instanceof Error ? error.message : String(error)}\`); + } }`; From 9e1f48a1274c00a4f5b851e0f81a6b6c2c377c26 Mon Sep 17 00:00:00 2001 From: drobnikj Date: Mon, 1 Dec 2025 13:01:21 +0100 Subject: [PATCH 10/18] fix: fix lint --- plugins/apify/index.tsx | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/plugins/apify/index.tsx b/plugins/apify/index.tsx index 7152bb54..80eac461 100644 --- a/plugins/apify/index.tsx +++ b/plugins/apify/index.tsx @@ -1,11 +1,8 @@ -import { Globe } from "lucide-react"; import type { IntegrationPlugin } from "../registry"; import { registerIntegration } from "../registry"; import { runActorCodegenTemplate } from "./codegen/run-actor"; import { scrapeSingleUrlCodegenTemplate } from "./codegen/scrape-single-url"; import { ApifyIcon } from "./icon"; -import { ApifySettings } from "./settings"; -import { testApify } from "./test"; const apifyPlugin: IntegrationPlugin = { type: "apify", @@ -14,8 +11,6 @@ const apifyPlugin: IntegrationPlugin = { icon: ApifyIcon, - settingsComponent: ApifySettings, - formFields: [ { id: "apifyApiKey", @@ -23,6 +18,7 @@ const apifyPlugin: IntegrationPlugin = { type: "password", placeholder: "apify_api_...", configKey: "apifyApiKey", + envVar: "APIFY_API_KEY", helpText: "Get your API token from ", helpLink: { text: "Apify Console", @@ -31,16 +27,11 @@ const apifyPlugin: IntegrationPlugin = { }, ], - credentialMapping: (config) => { - const creds: Record = {}; - if (config.apifyApiKey) { - creds.APIFY_API_KEY = String(config.apifyApiKey); - } - return creds; - }, - testConfig: { - testFunction: testApify, + getTestFunction: async () => { + const { testApify } = await import("./test"); + return testApify; + }, }, actions: [ @@ -49,7 +40,6 @@ const apifyPlugin: IntegrationPlugin = { label: "Run Apify Actor", description: "Run an Apify Actor and get results", category: "Apify", - icon: ApifyIcon, stepFunction: "apifyRunActorStep", stepImportPath: "run-actor/step", configFields: [ @@ -78,7 +68,6 @@ const apifyPlugin: IntegrationPlugin = { label: "Scrape Single URL", description: "Scrape a single URL and get markdown output", category: "Apify", - icon: Globe, stepFunction: "scrapeSingleUrlStep", stepImportPath: "scrape-single-url/step", configFields: [ From dacaa2459d7921d61b77fc936d7dca856cb2bba9 Mon Sep 17 00:00:00 2001 From: drobnikj Date: Mon, 1 Dec 2025 13:54:18 +0100 Subject: [PATCH 11/18] fix: remove spaces to reduce diff --- components/ui/drawer.tsx | 8 +------- components/ui/dropdown-menu.tsx | 8 +------- components/ui/tabs.tsx | 1 - components/workflow/config/action-config-renderer.tsx | 6 ------ components/workflow/config/action-config.tsx | 1 - lib/api-client.ts | 1 + lib/credential-fetcher.ts | 1 + lib/db/integrations.ts | 1 + lib/steps/index.ts | 1 - package.json | 3 +-- plugins/apify/index.tsx | 2 +- 11 files changed, 7 insertions(+), 26 deletions(-) diff --git a/components/ui/drawer.tsx b/components/ui/drawer.tsx index d21227cc..8aa69230 100644 --- a/components/ui/drawer.tsx +++ b/components/ui/drawer.tsx @@ -8,13 +8,7 @@ import { cn } from "@/lib/utils" function Drawer({ ...props }: React.ComponentProps) { - return ( - - ); + return } function DrawerTrigger({ diff --git a/components/ui/dropdown-menu.tsx b/components/ui/dropdown-menu.tsx index d615bfc6..4246c7d9 100644 --- a/components/ui/dropdown-menu.tsx +++ b/components/ui/dropdown-menu.tsx @@ -9,13 +9,7 @@ import { cn } from "@/lib/utils" function DropdownMenu({ ...props }: React.ComponentProps) { - return ( - - ); + return } function DropdownMenuPortal({ diff --git a/components/ui/tabs.tsx b/components/ui/tabs.tsx index 0f6ea6cd..cf2d1907 100644 --- a/components/ui/tabs.tsx +++ b/components/ui/tabs.tsx @@ -13,7 +13,6 @@ function Tabs({ ); diff --git a/components/workflow/config/action-config-renderer.tsx b/components/workflow/config/action-config-renderer.tsx index e155abbc..030c20ba 100644 --- a/components/workflow/config/action-config-renderer.tsx +++ b/components/workflow/config/action-config-renderer.tsx @@ -224,12 +224,6 @@ export function ActionConfigRenderer({ onUpdateConfig, disabled, }: ActionConfigRendererProps) { - // Safety check: ensure fields is an array - if (!Array.isArray(fields)) { - console.warn("ActionConfigRenderer: fields must be an array", fields); - return null; - } - return ( <> {fields.map((field) => { diff --git a/components/workflow/config/action-config.tsx b/components/workflow/config/action-config.tsx index 5b13ebbf..3931cf05 100644 --- a/components/workflow/config/action-config.tsx +++ b/components/workflow/config/action-config.tsx @@ -19,7 +19,6 @@ import { getAllIntegrations, } from "@/plugins"; import { ActionConfigRenderer } from "./action-config-renderer"; - import { SchemaBuilder, type SchemaField } from "./schema-builder"; type ActionConfigProps = { diff --git a/lib/api-client.ts b/lib/api-client.ts index f4ba676f..267aaf93 100644 --- a/lib/api-client.ts +++ b/lib/api-client.ts @@ -310,6 +310,7 @@ export const aiApi = { } }, }; + export type Integration = { id: string; name: string; diff --git a/lib/credential-fetcher.ts b/lib/credential-fetcher.ts index 430acb85..9301d5ae 100644 --- a/lib/credential-fetcher.ts +++ b/lib/credential-fetcher.ts @@ -18,6 +18,7 @@ import "server-only"; import { getCredentialMapping, getIntegration } from "@/plugins"; import { getIntegrationById } from "./db/integrations"; import type { IntegrationConfig, IntegrationType } from "./types/integration"; + // WorkflowCredentials is now a generic record since plugins define their own keys export type WorkflowCredentials = Record; diff --git a/lib/db/integrations.ts b/lib/db/integrations.ts index 139afd47..9ae2640e 100644 --- a/lib/db/integrations.ts +++ b/lib/db/integrations.ts @@ -94,6 +94,7 @@ function decryptConfig(encryptedConfig: string): Record { return {}; } } + export type DecryptedIntegration = { id: string; userId: string; diff --git a/lib/steps/index.ts b/lib/steps/index.ts index a26c601c..14269795 100644 --- a/lib/steps/index.ts +++ b/lib/steps/index.ts @@ -13,7 +13,6 @@ import type { firecrawlSearchStep } from "../../plugins/firecrawl/steps/search"; import type { createTicketStep } from "../../plugins/linear/steps/create-ticket"; import type { sendEmailStep } from "../../plugins/resend/steps/send-email"; import type { sendSlackMessageStep } from "../../plugins/slack/steps/send-slack-message"; - import type { conditionStep } from "./condition"; import type { databaseQueryStep } from "./database-query"; import type { httpRequestStep } from "./http-request"; diff --git a/package.json b/package.json index 6bfb35c7..94ebdeff 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,5 @@ "tw-animate-css": "^1.4.0", "typescript": "^5", "ultracite": "6.3.0" - }, - "packageManager": "pnpm@10.6.1+sha512.40ee09af407fa9fbb5fbfb8e1cb40fbb74c0af0c3e10e9224d7b53c7658528615b2c92450e74cfad91e3a2dcafe3ce4050d80bda71d757756d2ce2b66213e9a3" + } } diff --git a/plugins/apify/index.tsx b/plugins/apify/index.tsx index 80eac461..bd2b8935 100644 --- a/plugins/apify/index.tsx +++ b/plugins/apify/index.tsx @@ -83,7 +83,7 @@ const apifyPlugin: IntegrationPlugin = { key: "crawlerType", label: "Crawler Type", type: "select", - defaultValue: "playwright", + defaultValue: "playwright:adaptive", options: [ { value: "playwright:adaptive", From dc0ea28aab8192f5fec826ea5e0caf228662e1d7 Mon Sep 17 00:00:00 2001 From: drobnikj Date: Mon, 1 Dec 2025 15:49:09 +0100 Subject: [PATCH 12/18] fix: remove debugging logs --- lib/workflow-executor.workflow.ts | 33 ------------------------------- 1 file changed, 33 deletions(-) diff --git a/lib/workflow-executor.workflow.ts b/lib/workflow-executor.workflow.ts index b02e9435..8b4958bd 100644 --- a/lib/workflow-executor.workflow.ts +++ b/lib/workflow-executor.workflow.ts @@ -241,39 +241,6 @@ async function executeActionStep(input: { }; } - if (actionType === "Run Apify Actor") { - const { apifyRunActorStep } = await import( - "../plugins/apify/steps/run-actor/step" - ); - const runActorInput = { ...stepInput }; - if (typeof runActorInput.actorInput === "string") { - try { - runActorInput.actorInput = JSON.parse(runActorInput.actorInput); - } catch { - // If JSON parsing fails, keep the original string - console.warn("[Run Apify Actor] Failed to parse actorInput as JSON"); - } - } - console.log( - "[Run Apify Actor] Input:", - JSON.stringify(runActorInput, null, 2) - ); - // biome-ignore lint/suspicious/noExplicitAny: Dynamic step input type - return await apifyRunActorStep(runActorInput as any); - } - - if (actionType === "Scrape Single URL") { - const { scrapeSingleUrlStep } = await import( - "../plugins/apify/steps/scrape-single-url/step" - ); - console.log( - "[Scrape Single URL] Input:", - JSON.stringify(stepInput, null, 2) - ); - // biome-ignore lint/suspicious/noExplicitAny: Dynamic step input type - return await scrapeSingleUrlStep(stepInput as any); - } - // Fallback for unknown action types return { success: false, From 9ec9d3517197c0fe9f2f8a5637f1b94721149113 Mon Sep 17 00:00:00 2001 From: drobnikj Date: Mon, 1 Dec 2025 16:32:30 +0100 Subject: [PATCH 13/18] fix: clean up code, add JSON error parsing --- components/ui/template-badge-json.tsx | 145 ++++++++++++++++++ components/workflow/config/action-config.tsx | 25 ++- plugins/apify/codegen/run-actor.ts | 24 +-- plugins/apify/codegen/scrape-single-url.ts | 10 +- plugins/apify/steps/run-actor/config.tsx | 26 ++-- plugins/apify/steps/run-actor/step.ts | 90 ++++++----- .../apify/steps/scrape-single-url/config.tsx | 62 -------- plugins/apify/steps/scrape-single-url/step.ts | 128 ++++++++-------- 8 files changed, 316 insertions(+), 194 deletions(-) create mode 100644 components/ui/template-badge-json.tsx delete mode 100644 plugins/apify/steps/scrape-single-url/config.tsx diff --git a/components/ui/template-badge-json.tsx b/components/ui/template-badge-json.tsx new file mode 100644 index 00000000..b67492fd --- /dev/null +++ b/components/ui/template-badge-json.tsx @@ -0,0 +1,145 @@ +"use client"; + +import { useEffect, useRef, useState } from "react"; +import { cn } from "@/lib/utils"; +import { TemplateBadgeTextarea } from "./template-badge-textarea"; + +export interface TemplateBadgeJsonProps { + value?: string; + onChange?: (value: string) => void; + placeholder?: string; + disabled?: boolean; + className?: string; + id?: string; + rows?: number; +} + +/** + * A textarea component that validates JSON input in real-time + * Wraps TemplateBadgeTextarea and adds JSON validation and formatting + */ +export function TemplateBadgeJson({ + value = "", + onChange, + placeholder, + disabled, + className, + id, + rows = 3, +}: TemplateBadgeJsonProps) { + const [jsonError, setJsonError] = useState(null); + const [isFocused, setIsFocused] = useState(false); + const formatTimeoutRef = useRef(null); + const lastFormattedValueRef = useRef(""); + + // Validate JSON on value change + useEffect(() => { + if (!value || typeof value !== "string") { + setJsonError(null); + return; + } + + // If empty or only whitespace, no error + if (!value.trim()) { + setJsonError(null); + return; + } + + // Parse JSON directly - template variables will be treated as normal strings + try { + JSON.parse(value); + setJsonError(null); + } catch (error) { + setJsonError( + error instanceof Error ? error.message : "Invalid JSON format" + ); + } + }, [value]); + + // Format JSON when it becomes valid (debounced to avoid formatting while typing) + useEffect(() => { + // Clear any pending format timeout + if (formatTimeoutRef.current) { + clearTimeout(formatTimeoutRef.current); + } + + // Don't format if there's an error, field is focused, or value is empty + if (jsonError || isFocused || !value || typeof value !== "string") { + return; + } + + if (!value.trim()) { + return; + } + + // Debounce formatting - wait 500ms after user stops typing + formatTimeoutRef.current = setTimeout(() => { + try { + // Parse JSON directly - template variables are treated as normal strings + const parsed = JSON.parse(value); + const formatted = JSON.stringify(parsed, null, 2); + + // Only format if different from current value and we haven't already formatted this value + if (formatted !== value && formatted !== lastFormattedValueRef.current) { + lastFormattedValueRef.current = formatted; + onChange?.(formatted); + } + } catch { + // If parsing fails, don't format + } + }, 500); + + return () => { + if (formatTimeoutRef.current) { + clearTimeout(formatTimeoutRef.current); + } + }; + }, [value, isFocused, jsonError, onChange]); + + // Track focus state by listening to focus/blur events on the wrapper + const handleWrapperFocus = () => { + setIsFocused(true); + }; + + const handleWrapperBlur = () => { + setIsFocused(false); + // Format immediately on blur if JSON is valid + if (!jsonError && value && typeof value === "string" && value.trim()) { + try { + // Parse JSON directly - template variables are treated as normal strings + const parsed = JSON.parse(value); + const formatted = JSON.stringify(parsed, null, 2); + + if (formatted !== value) { + onChange?.(formatted); + } + } catch { + // If parsing fails, don't format + } + } + }; + + return ( +
+ + {jsonError && ( +

{jsonError}

+ )} +
+ ); +} diff --git a/components/workflow/config/action-config.tsx b/components/workflow/config/action-config.tsx index 3931cf05..152d01f0 100644 --- a/components/workflow/config/action-config.tsx +++ b/components/workflow/config/action-config.tsx @@ -18,6 +18,7 @@ import { getActionsByCategory, getAllIntegrations, } from "@/plugins"; +import { RunActorConfigFields } from "@/plugins/apify/steps/run-actor/config"; import { ActionConfigRenderer } from "./action-config-renderer"; import { SchemaBuilder, type SchemaField } from "./schema-builder"; @@ -390,14 +391,22 @@ export function ActionConfig({ )} {/* Plugin actions - declarative config fields */} - {pluginAction && !SYSTEM_ACTION_IDS.includes(actionType) && ( - - )} + {pluginAction && + !SYSTEM_ACTION_IDS.includes(actionType) && + (actionType === "apify/run-actor" ? ( + + ) : ( + + ))} ); } diff --git a/plugins/apify/codegen/run-actor.ts b/plugins/apify/codegen/run-actor.ts index 6d953db9..826c0aea 100644 --- a/plugins/apify/codegen/run-actor.ts +++ b/plugins/apify/codegen/run-actor.ts @@ -7,7 +7,7 @@ export const runActorCodegenTemplate = `import { ApifyClient } from "apify-clien export async function apifyRunActorStep(input: { actorId: string; - actorInput?: Record; + actorInput?: string; }) { "use step"; @@ -17,30 +17,36 @@ export async function apifyRunActorStep(input: { throw new Error("Apify API Token is not configured. Set APIFY_API_KEY environment variable."); } + let parsedActorInput: Record = {}; + if (input.actorInput) { + try { + parsedActorInput = JSON.parse(input.actorInput); + } catch (err) { + throw new Error(\`Cannot parse Actor input: \${err instanceof Error ? err.message : String(err)}\`); + } + } + try { const client = new ApifyClient({ token: apiKey }); const actorClient = client.actor(input.actorId); - const maxWaitSecs = 120; // Run synchronously and wait for completion - const runData = await actorClient.call(input.actorInput || {}, { - waitSecs: maxWaitSecs, - }); + const runData = await actorClient.call(parsedActorInput); // Get dataset items - let data: unknown[] = []; + let datasetItems: unknown[] = []; if (runData.defaultDatasetId) { - const datasetItems = await client + const dataset = await client .dataset(runData.defaultDatasetId) .listItems(); - data = datasetItems.items; + datasetItems = dataset.items; } return { runId: runData.id || "unknown", status: runData.status || "SUCCEEDED", datasetId: runData.defaultDatasetId, - data, + datasetItems, }; } catch (error) { throw new Error(\`Failed to run Actor: \${error instanceof Error ? error.message : String(error)}\`); diff --git a/plugins/apify/codegen/scrape-single-url.ts b/plugins/apify/codegen/scrape-single-url.ts index 2b1b026a..ec35c6c8 100644 --- a/plugins/apify/codegen/scrape-single-url.ts +++ b/plugins/apify/codegen/scrape-single-url.ts @@ -24,7 +24,7 @@ export async function scrapeSingleUrlStep(input: { try { const client = new ApifyClient({ token: apiKey }); const actorClient = client.actor("apify/website-content-crawler"); - const crawlerType = input.crawlerType || "playwright"; + const crawlerType = input.crawlerType || "playwright:adaptive"; // Prepare actor input const actorInput = { @@ -37,15 +37,11 @@ export async function scrapeSingleUrlStep(input: { useApifyProxy: true, }, removeCookieWarnings: true, - saveHtml: true, saveMarkdown: true, }; // Run synchronously and wait for completion - const maxWaitSecs = 120; - const runData = await actorClient.call(actorInput, { - waitSecs: maxWaitSecs, - }); + const runData = await actorClient.call(actorInput); // Get dataset items let markdown: string | undefined; @@ -57,7 +53,7 @@ export async function scrapeSingleUrlStep(input: { // Extract markdown from the first item if (datasetItems.items && datasetItems.items.length > 0) { const firstItem = datasetItems.items[0] as Record; - markdown = (firstItem.markdown as string) || (firstItem.text as string) || undefined; + markdown = firstItem.markdown as string; } } diff --git a/plugins/apify/steps/run-actor/config.tsx b/plugins/apify/steps/run-actor/config.tsx index 1ed183e2..b34ad89b 100644 --- a/plugins/apify/steps/run-actor/config.tsx +++ b/plugins/apify/steps/run-actor/config.tsx @@ -1,6 +1,6 @@ import { Label } from "@/components/ui/label"; import { TemplateBadgeInput } from "@/components/ui/template-badge-input"; -import { TemplateBadgeTextarea } from "@/components/ui/template-badge-textarea"; +import { TemplateBadgeJson } from "@/components/ui/template-badge-json"; /** * Run Apify Actor Config Fields Component @@ -27,20 +27,26 @@ export function RunActorConfigFields({ value={(config?.actorId as string) || ""} />

- Enter the Actor ID (e.g., apify/web-scraper) or use a template - reference. + Enter an Actor ID or name (e.g., apify/website-content-crawler). Browse all available Actors in the + Apify Store + .

- onUpdateConfig("actorInput", value)} - value={(config?.actorInput as string) || ""} + onUpdateConfig("actorInput", value)} + value={(config?.actorInput as string) || ""} />

JSON input for the Actor. Check the Actor's documentation for required diff --git a/plugins/apify/steps/run-actor/step.ts b/plugins/apify/steps/run-actor/step.ts index 89c404bf..610261d0 100644 --- a/plugins/apify/steps/run-actor/step.ts +++ b/plugins/apify/steps/run-actor/step.ts @@ -3,6 +3,7 @@ import "server-only"; import { ApifyClient } from "apify-client"; import { fetchCredentials } from "@/lib/credential-fetcher"; import { getErrorMessage } from "@/lib/utils"; +import { type StepInput, withStepLogging } from "@/lib/steps/step-handler"; type ApifyRunActorResult = | { @@ -18,56 +19,69 @@ type ApifyRunActorResult = * Run Apify Actor Step * Runs an Apify Actor and optionally waits for results */ -export async function apifyRunActorStep(input: { - integrationId?: string; - actorId: string; - actorInput?: Record; -}): Promise { +export async function apifyRunActorStep( + input: { + integrationId?: string; + actorId: string; + actorInput?: string; + } & StepInput +): Promise { "use step"; - const credentials = input.integrationId - ? await fetchCredentials(input.integrationId) - : {}; + return withStepLogging(input, async () => { + const credentials = input.integrationId + ? await fetchCredentials(input.integrationId) + : {}; - const apiKey = credentials.APIFY_API_KEY; + const apiKey = credentials.APIFY_API_KEY; - if (!apiKey) { - return { - success: false, - error: "Apify API Token is not configured.", - }; - } + if (!apiKey) { + return { + success: false, + error: "Apify API Token is not configured.", + }; + } + + let parsedActorInput = {}; + if (input?.actorInput) { + try { + parsedActorInput = JSON.parse(input?.actorInput); + } catch (err) { + return { + success: false, + error: `Cannot parse Actor input: ${getErrorMessage(err)}`, + }; + } + } - try { - const client = new ApifyClient({ token: apiKey }); - const actorClient = client.actor(input.actorId); - const maxWaitSecs = 120; + try { + const client = new ApifyClient({ token: apiKey }); + const actorClient = client.actor(input.actorId); // Run synchronously and wait for completion - const runData = await actorClient.call(input.actorInput || {}, { - waitSecs: maxWaitSecs, - }); + const runData = await actorClient.call(parsedActorInput); // Get dataset items - let data: unknown[] = []; + let datasetItems: unknown[] = []; if (runData.defaultDatasetId) { - const datasetItems = await client - .dataset(runData.defaultDatasetId) - .listItems(); - data = datasetItems.items; + const dataset = await client + .dataset(runData.defaultDatasetId) + .listItems(); + datasetItems = dataset.items; } return { - success: true, - runId: runData.id || "unknown", - status: runData.status || "SUCCEEDED", - datasetId: runData.defaultDatasetId, - data, + success: true, + runId: runData.id || "unknown", + status: runData.status || "SUCCEEDED", + datasetId: runData.defaultDatasetId, + datasetItems, + }; + } catch (error) { + return { + success: false, + error: `Failed to run Actor: ${getErrorMessage(error)}`, }; - } catch (error) { - return { - success: false, - error: `Failed to run Actor: ${getErrorMessage(error)}`, - }; - } + } + }); } diff --git a/plugins/apify/steps/scrape-single-url/config.tsx b/plugins/apify/steps/scrape-single-url/config.tsx deleted file mode 100644 index e3c0db30..00000000 --- a/plugins/apify/steps/scrape-single-url/config.tsx +++ /dev/null @@ -1,62 +0,0 @@ -import { Label } from "@/components/ui/label"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "@/components/ui/select"; -import { TemplateBadgeInput } from "@/components/ui/template-badge-input"; - -/** - * Scrape Single URL Config Fields Component - * UI for configuring the scrape single URL action - */ -export function ScrapeSingleUrlConfigFields({ - config, - onUpdateConfig, - disabled, -}: { - config: Record; - onUpdateConfig: (key: string, value: unknown) => void; - disabled?: boolean; -}) { - return ( -

-
- - onUpdateConfig("url", value)} - placeholder="https://example.com or {{NodeName.url}}" - value={(config?.url as string) || ""} - /> -

- Enter the URL to scrape or use a template reference. -

-
- -
- - -

- Select the crawler type to use for scraping. -

-
-
- ); -} diff --git a/plugins/apify/steps/scrape-single-url/step.ts b/plugins/apify/steps/scrape-single-url/step.ts index 5adf2945..3b0bb7b7 100644 --- a/plugins/apify/steps/scrape-single-url/step.ts +++ b/plugins/apify/steps/scrape-single-url/step.ts @@ -3,6 +3,7 @@ import "server-only"; import { ApifyClient } from "apify-client"; import { fetchCredentials } from "@/lib/credential-fetcher"; import { getErrorMessage } from "@/lib/utils"; +import { type StepInput, withStepLogging } from "@/lib/steps/step-handler"; type ScrapeSingleUrlResult = | { @@ -17,83 +18,90 @@ type ScrapeSingleUrlResult = * Scrape Single URL Step * Scrapes a single URL using apify/website-content-crawler and returns markdown */ -export async function scrapeSingleUrlStep(input: { - integrationId?: string; - url: string; - crawlerType?: string; -}): Promise { +export async function scrapeSingleUrlStep( + input: { + integrationId?: string; + url: string; + crawlerType?: string; + } & StepInput +): Promise { "use step"; - const credentials = input.integrationId - ? await fetchCredentials(input.integrationId) - : {}; + return withStepLogging(input, async () => { + const credentials = input.integrationId + ? await fetchCredentials(input.integrationId) + : {}; - const apiKey = credentials.APIFY_API_KEY; + const apiKey = credentials.APIFY_API_KEY; - if (!apiKey) { - return { - success: false, - error: "Apify API Token is not configured.", - }; - } + if (!apiKey) { + return { + success: false, + error: "Apify API Token is not configured.", + }; + } - if (!input.url) { - return { - success: false, - error: "URL is required.", - }; - } + if (!input.url) { + return { + success: false, + error: "URL is required.", + }; + } - try { - const client = new ApifyClient({ token: apiKey }); - const actorClient = client.actor("apify/website-content-crawler"); - const crawlerType = input.crawlerType || "playwright"; + try { + const client = new ApifyClient({ token: apiKey }); + const actorClient = client.actor("apify/website-content-crawler"); + const crawlerType = input.crawlerType || "playwright:adaptive"; - // Prepare actor input - const actorInput = { - startUrls: [{ url: input.url }], - crawlerType, + // Prepare actor input + const actorInput = { + startUrls: [{ url: input.url }], + crawlerType, maxCrawlDepth: 0, maxCrawlPages: 1, maxResults: 1, proxyConfiguration: { - useApifyProxy: true, + useApifyProxy: true, }, removeCookieWarnings: true, - saveHtml: true, saveMarkdown: true, - }; + }; - // Run synchronously and wait for completion - const maxWaitSecs = 120; - const runData = await actorClient.call(actorInput, { - waitSecs: maxWaitSecs, - }); + // Run synchronously and wait for completion (waits indefinitely if waitSecs not specified) + const runData = await actorClient.call(actorInput); + console.log("[Scrape Single URL] Actor call completed:", { + runId: runData.id, + status: runData.status, + hasDataset: !!runData.defaultDatasetId, + }); - // Get dataset items - let markdown: string | undefined; - if (runData.defaultDatasetId) { - const datasetItems = await client - .dataset(runData.defaultDatasetId) - .listItems(); + // Get dataset items + let markdown: string | undefined; + if (runData.defaultDatasetId) { + const datasetItems = await client + .dataset(runData.defaultDatasetId) + .listItems(); - // Extract markdown from the first item - if (datasetItems.items && datasetItems.items.length > 0) { - const firstItem = datasetItems.items[0] as Record; - markdown = (firstItem.markdown as string) || (firstItem.text as string) || undefined; + // Extract markdown from the first item + if (datasetItems.items && datasetItems.items.length > 0) { + const firstItem = datasetItems.items[0] as Record; + markdown = (firstItem.markdown as string); + } } - } - return { - success: true, - runId: runData.id || "unknown", - status: runData.status || "SUCCEEDED", - markdown, - }; - } catch (error) { - return { - success: false, - error: `Failed to scrape URL: ${getErrorMessage(error)}`, - }; - } + const result: ScrapeSingleUrlResult = { + success: true, + runId: runData.id || "unknown", + status: runData.status || "SUCCEEDED", + markdown, + }; + + return result; + } catch (error) { + return { + success: false, + error: `Failed to scrape URL: ${getErrorMessage(error)}`, + }; + } + }); } From fce3519e32354acd73617ca245a74495e7efcea6 Mon Sep 17 00:00:00 2001 From: drobnikj Date: Wed, 3 Dec 2025 11:29:07 +0100 Subject: [PATCH 14/18] fix: review comments fix --- README.md | 2 +- .../[integrationId]/test/route.ts | 2 +- components/ui/template-badge-json.tsx | 6 +++ lib/step-registry.ts | 2 +- lib/steps/index.ts | 2 +- plugins/apify/codegen/run-actor.ts | 8 ++-- plugins/apify/codegen/scrape-single-url.ts | 4 +- plugins/apify/icon.tsx | 2 +- plugins/apify/index.tsx | 14 +++--- plugins/apify/settings.tsx | 47 ------------------- plugins/apify/steps/run-actor/config.tsx | 2 +- plugins/apify/steps/run-actor/step.ts | 6 +-- plugins/apify/steps/scrape-single-url/step.ts | 7 +-- plugins/apify/test.ts | 2 +- 14 files changed, 30 insertions(+), 76 deletions(-) delete mode 100644 plugins/apify/settings.tsx diff --git a/README.md b/README.md index c8d580e2..d0bb35ec 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ Visit [http://localhost:3000](http://localhost:3000) to get started. - **AI Gateway**: Generate Text, Generate Image -- **Apify**: Run Apify Actor, Scrape Single URL +- **Apify**: Run Actor, Scrape Single URL - **Firecrawl**: Scrape URL, Search Web - **Linear**: Create Ticket, Find Issues - **Resend**: Send Email diff --git a/app/api/integrations/[integrationId]/test/route.ts b/app/api/integrations/[integrationId]/test/route.ts index 1af52776..3085f617 100644 --- a/app/api/integrations/[integrationId]/test/route.ts +++ b/app/api/integrations/[integrationId]/test/route.ts @@ -70,7 +70,7 @@ export async function POST( ); break; case "apify": - result = await testApifyConnection(integration.config.apifyApiKey); + result = await testApifyConnection(integration.config.apifyApiToken); break; default: return NextResponse.json( diff --git a/components/ui/template-badge-json.tsx b/components/ui/template-badge-json.tsx index b67492fd..223a88a2 100644 --- a/components/ui/template-badge-json.tsx +++ b/components/ui/template-badge-json.tsx @@ -45,6 +45,12 @@ export function TemplateBadgeJson({ return; } + // Ensure that parsable values (not object) throws + if (!/^\s*\{[\s\S]*\}\s*$/.test(value)) { + setJsonError("Value must be a JSON object"); + return; + } + // Parse JSON directly - template variables will be treated as normal strings try { JSON.parse(value); diff --git a/lib/step-registry.ts b/lib/step-registry.ts index 6e51f4a2..48876bc9 100644 --- a/lib/step-registry.ts +++ b/lib/step-registry.ts @@ -122,7 +122,7 @@ export const PLUGIN_STEP_IMPORTERS: Record = { export const ACTION_LABELS: Record = { "ai-gateway/generate-text": "Generate Text", "ai-gateway/generate-image": "Generate Image", - "apify/run-actor": "Run Apify Actor", + "apify/run-actor": "Run Actor", "apify/scrape-single-url": "Scrape Single URL", "firecrawl/scrape": "Scrape URL", "firecrawl/search": "Search Web", diff --git a/lib/steps/index.ts b/lib/steps/index.ts index 14269795..32959688 100644 --- a/lib/steps/index.ts +++ b/lib/steps/index.ts @@ -66,7 +66,7 @@ export const stepRegistry: Record = { (await import("../../plugins/firecrawl/steps/search")).firecrawlSearchStep( input as Parameters[0] ), - "Run Apify Actor": async (input) => + "Run Actor": async (input) => ( await import("../../plugins/apify/steps/run-actor/step") ).apifyRunActorStep(input as Parameters[0]), diff --git a/plugins/apify/codegen/run-actor.ts b/plugins/apify/codegen/run-actor.ts index 826c0aea..aa43bb55 100644 --- a/plugins/apify/codegen/run-actor.ts +++ b/plugins/apify/codegen/run-actor.ts @@ -1,5 +1,5 @@ /** - * Code generation template for Run Apify Actor action + * Code generation template for Run Actor action * This template is used when exporting workflows to standalone Next.js projects * It uses environment variables instead of integrationId */ @@ -11,10 +11,10 @@ export async function apifyRunActorStep(input: { }) { "use step"; - const apiKey = process.env.APIFY_API_KEY; + const apiKey = process.env.APIFY_API_TOKEN; if (!apiKey) { - throw new Error("Apify API Token is not configured. Set APIFY_API_KEY environment variable."); + throw new Error("Apify API Token is not configured. Set APIFY_API_TOKEN environment variable."); } let parsedActorInput: Record = {}; @@ -46,7 +46,7 @@ export async function apifyRunActorStep(input: { runId: runData.id || "unknown", status: runData.status || "SUCCEEDED", datasetId: runData.defaultDatasetId, - datasetItems, + data: datasetItems, }; } catch (error) { throw new Error(\`Failed to run Actor: \${error instanceof Error ? error.message : String(error)}\`); diff --git a/plugins/apify/codegen/scrape-single-url.ts b/plugins/apify/codegen/scrape-single-url.ts index ec35c6c8..cc4d5d82 100644 --- a/plugins/apify/codegen/scrape-single-url.ts +++ b/plugins/apify/codegen/scrape-single-url.ts @@ -11,10 +11,10 @@ export async function scrapeSingleUrlStep(input: { }) { "use step"; - const apiKey = process.env.APIFY_API_KEY; + const apiKey = process.env.APIFY_API_TOKEN; if (!apiKey) { - throw new Error("Apify API Token is not configured. Set APIFY_API_KEY environment variable."); + throw new Error("Apify API Token is not configured. Set APIFY_API_TOKEN environment variable."); } if (!input.url) { diff --git a/plugins/apify/icon.tsx b/plugins/apify/icon.tsx index 26358944..6329437e 100644 --- a/plugins/apify/icon.tsx +++ b/plugins/apify/icon.tsx @@ -2,7 +2,7 @@ import Image from "next/image"; /** * Apify Icon Component - * Used as the icon for Run Apify Actor action + * Used as the icon for Run Actor action */ export function ApifyIcon({ className }: { className?: string }) { return ( diff --git a/plugins/apify/index.tsx b/plugins/apify/index.tsx index bd2b8935..faab5338 100644 --- a/plugins/apify/index.tsx +++ b/plugins/apify/index.tsx @@ -13,12 +13,12 @@ const apifyPlugin: IntegrationPlugin = { formFields: [ { - id: "apifyApiKey", - label: "API Token", + id: "apifyApiToken", + label: "Apify API Token", type: "password", placeholder: "apify_api_...", - configKey: "apifyApiKey", - envVar: "APIFY_API_KEY", + configKey: "apifyApiToken", + envVar: "APIFY_API_TOKEN", helpText: "Get your API token from ", helpLink: { text: "Apify Console", @@ -37,7 +37,7 @@ const apifyPlugin: IntegrationPlugin = { actions: [ { slug: "run-actor", - label: "Run Apify Actor", + label: "Run Actor", description: "Run an Apify Actor and get results", category: "Apify", stepFunction: "apifyRunActorStep", @@ -47,8 +47,8 @@ const apifyPlugin: IntegrationPlugin = { key: "actorId", label: "Actor (ID or name)", type: "template-input", - placeholder: "apify/web-scraper or {{NodeName.actorId}}", - example: "apify/web-scraper", + placeholder: "apify/website-content-crawler or {{NodeName.actorId}}", + example: "apify/website-content-crawler", required: true, }, { diff --git a/plugins/apify/settings.tsx b/plugins/apify/settings.tsx deleted file mode 100644 index 932f9a56..00000000 --- a/plugins/apify/settings.tsx +++ /dev/null @@ -1,47 +0,0 @@ -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; - -export function ApifySettings({ - apiKey, - hasKey, - onApiKeyChange, -}: { - apiKey: string; - hasKey?: boolean; - onApiKeyChange: (key: string) => void; - showCard?: boolean; - config?: Record; - onConfigChange?: (key: string, value: string) => void; -}) { - return ( -
-
- - onApiKeyChange(e.target.value)} - placeholder={ - hasKey ? "API token is configured" : "Enter your Apify API token" - } - type="password" - value={apiKey} - /> -

- Get your API token from{" "} - - Apify Console - - . -

-
-
- ); -} diff --git a/plugins/apify/steps/run-actor/config.tsx b/plugins/apify/steps/run-actor/config.tsx index b34ad89b..e085da9e 100644 --- a/plugins/apify/steps/run-actor/config.tsx +++ b/plugins/apify/steps/run-actor/config.tsx @@ -3,7 +3,7 @@ import { TemplateBadgeInput } from "@/components/ui/template-badge-input"; import { TemplateBadgeJson } from "@/components/ui/template-badge-json"; /** - * Run Apify Actor Config Fields Component + * Run Actor Config Fields Component * UI for configuring the run actor action */ export function RunActorConfigFields({ diff --git a/plugins/apify/steps/run-actor/step.ts b/plugins/apify/steps/run-actor/step.ts index 610261d0..3541a6f3 100644 --- a/plugins/apify/steps/run-actor/step.ts +++ b/plugins/apify/steps/run-actor/step.ts @@ -16,7 +16,7 @@ type ApifyRunActorResult = | { success: false; error: string }; /** - * Run Apify Actor Step + * Run Actor Step * Runs an Apify Actor and optionally waits for results */ export async function apifyRunActorStep( @@ -33,7 +33,7 @@ export async function apifyRunActorStep( ? await fetchCredentials(input.integrationId) : {}; - const apiKey = credentials.APIFY_API_KEY; + const apiKey = credentials.APIFY_API_TOKEN; if (!apiKey) { return { @@ -75,7 +75,7 @@ export async function apifyRunActorStep( runId: runData.id || "unknown", status: runData.status || "SUCCEEDED", datasetId: runData.defaultDatasetId, - datasetItems, + data: datasetItems, }; } catch (error) { return { diff --git a/plugins/apify/steps/scrape-single-url/step.ts b/plugins/apify/steps/scrape-single-url/step.ts index 3b0bb7b7..09c9f966 100644 --- a/plugins/apify/steps/scrape-single-url/step.ts +++ b/plugins/apify/steps/scrape-single-url/step.ts @@ -32,7 +32,7 @@ export async function scrapeSingleUrlStep( ? await fetchCredentials(input.integrationId) : {}; - const apiKey = credentials.APIFY_API_KEY; + const apiKey = credentials.APIFY_API_TOKEN; if (!apiKey) { return { @@ -69,11 +69,6 @@ export async function scrapeSingleUrlStep( // Run synchronously and wait for completion (waits indefinitely if waitSecs not specified) const runData = await actorClient.call(actorInput); - console.log("[Scrape Single URL] Actor call completed:", { - runId: runData.id, - status: runData.status, - hasDataset: !!runData.defaultDatasetId, - }); // Get dataset items let markdown: string | undefined; diff --git a/plugins/apify/test.ts b/plugins/apify/test.ts index 8382720f..729bf786 100644 --- a/plugins/apify/test.ts +++ b/plugins/apify/test.ts @@ -2,7 +2,7 @@ import { ApifyClient } from "apify-client"; export async function testApify(credentials: Record) { try { - const client = new ApifyClient({ token: credentials.APIFY_API_KEY }); + const client = new ApifyClient({ token: credentials.APIFY_API_TOKEN }); await client.user("me").get(); return { success: true }; } catch (error) { From 3885666fb27c64f4ce1fe3e766e23ffd34b7185d Mon Sep 17 00:00:00 2001 From: drobnikj Date: Wed, 3 Dec 2025 11:49:07 +0100 Subject: [PATCH 15/18] fix: update readme --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index d0bb35ec..2c99c9ba 100644 --- a/README.md +++ b/README.md @@ -263,6 +263,30 @@ const searchResult = await firecrawlSearchStep({ }); ``` +### Apify (Web Scraping) + +```typescript +import { + scrapeSingleUrlStep, + apifyRunActorStep, +} from "@/lib/steps/apify"; + +// Scrape a URL +const scrapeResult = await scrapeSingleUrlStep({ + url: "https://example.com", + crawlerType: "playwright:adaptive", +}); + +// Run an Actor from Apify Store +const searchMapsResults = await apifyRunActorStep({ + actorId: "compass/crawler-google-places", + actorInput: { + searchStringsArray: [ "restaurants in San Francisco" ] + }, +}); +``` + + ## Tech Stack - **Framework**: Next.js 16 with React 19 From d391d21ac35f74d168c2db41fc11b2ba815f0850 Mon Sep 17 00:00:00 2001 From: drobnikj Date: Wed, 3 Dec 2025 12:00:41 +0100 Subject: [PATCH 16/18] fix: update apifyRunActorStep to accept object same as string --- plugins/apify/codegen/run-actor.ts | 16 +++++++++++----- plugins/apify/steps/run-actor/step.ts | 20 +++++++++++++------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/plugins/apify/codegen/run-actor.ts b/plugins/apify/codegen/run-actor.ts index aa43bb55..9306ef3f 100644 --- a/plugins/apify/codegen/run-actor.ts +++ b/plugins/apify/codegen/run-actor.ts @@ -7,7 +7,7 @@ export const runActorCodegenTemplate = `import { ApifyClient } from "apify-clien export async function apifyRunActorStep(input: { actorId: string; - actorInput?: string; + actorInput?: string | Record | null; }) { "use step"; @@ -19,10 +19,16 @@ export async function apifyRunActorStep(input: { let parsedActorInput: Record = {}; if (input.actorInput) { - try { - parsedActorInput = JSON.parse(input.actorInput); - } catch (err) { - throw new Error(\`Cannot parse Actor input: \${err instanceof Error ? err.message : String(err)}\`); + // If it's already an object, use it directly + if (typeof input.actorInput === "object" && !Array.isArray(input.actorInput)) { + parsedActorInput = input.actorInput; + } else if (typeof input.actorInput === "string") { + // If it's a string, parse it + try { + parsedActorInput = JSON.parse(input.actorInput); + } catch (err) { + throw new Error(\`Cannot parse Actor input: \${err instanceof Error ? err.message : String(err)}\`); + } } } diff --git a/plugins/apify/steps/run-actor/step.ts b/plugins/apify/steps/run-actor/step.ts index 3541a6f3..44a586f6 100644 --- a/plugins/apify/steps/run-actor/step.ts +++ b/plugins/apify/steps/run-actor/step.ts @@ -23,7 +23,7 @@ export async function apifyRunActorStep( input: { integrationId?: string; actorId: string; - actorInput?: string; + actorInput?: string | Record | null; } & StepInput ): Promise { "use step"; @@ -42,16 +42,22 @@ export async function apifyRunActorStep( }; } - let parsedActorInput = {}; + let parsedActorInput: Record = {}; if (input?.actorInput) { + // If it's already an object, use it directly + if (typeof input.actorInput === "object" && !Array.isArray(input.actorInput)) { + parsedActorInput = input.actorInput; + } else if (typeof input.actorInput === "string") { + // If it's a string, parse it try { - parsedActorInput = JSON.parse(input?.actorInput); + parsedActorInput = JSON.parse(input.actorInput); } catch (err) { - return { - success: false, - error: `Cannot parse Actor input: ${getErrorMessage(err)}`, - }; + return { + success: false, + error: `Cannot parse Actor input: ${getErrorMessage(err)}`, + }; } + } } try { From eb901028b0781f3004f01a8a4f3344635399599e Mon Sep 17 00:00:00 2001 From: drobnikj Date: Thu, 4 Dec 2025 11:52:02 +0100 Subject: [PATCH 17/18] fix: resolving merge conflicts --- pnpm-lock.yaml | 114 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0c7d892f..b3574029 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -50,6 +50,9 @@ importers: ai: specifier: ^5.0.102 version: 5.0.102(zod@4.1.12) + apify-client: + specifier: ^2.20.0 + version: 2.20.0 better-auth: specifier: ^1.3.34 version: 1.3.34(next@16.0.1(@babel/core@7.28.5)(@opentelemetry/api@1.9.0)(@playwright/test@1.57.0)(react-dom@19.2.1(react@19.2.1))(react@19.2.1))(react-dom@19.2.1(react@19.2.1))(react@19.2.1) @@ -194,6 +197,15 @@ packages: resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} + '@apify/consts@2.48.0': + resolution: {integrity: sha512-a0HeYDxAbbkRxc9z2N6beMFAmAJSgBw8WuKUwV+KmCuPyGUVLp54fYzjQ63p9Gv5IVFC88/HMXpAzI29ARgO5w==} + + '@apify/log@2.5.28': + resolution: {integrity: sha512-jU8qIvU+Crek8glBjFl3INjJQWWDR9n2z9Dr0WvUI8KJi0LG9fMdTvV+Aprf9z1b37CbHXgiZkA1iPlNYxKOEQ==} + + '@apify/utilities@2.24.0': + resolution: {integrity: sha512-tc1lH0T4C0KOrCFKy4+pP/bj0VkAKUOqG1Unyyz0ddKv2b8R1PKxboEx2sGDZSHxCLA+nbk/kC2jgIvGqen2KA==} + '@aws-crypto/sha256-browser@5.2.0': resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==} @@ -413,6 +425,10 @@ packages: '@clack/prompts@0.11.0': resolution: {integrity: sha512-pMN5FcrEw9hUkZA4f+zLlzivQSeQf5dRGJjSUbvVYDLvpKCdQx5OaknvKzgbtXOizhP+SJJJjqEbOe55uKKfAw==} + '@crawlee/types@3.15.3': + resolution: {integrity: sha512-RvgVPXrsQw4GQIUXrC1z1aNOedUPJnZ/U/8n+jZ0fu1Iw9moJVMuiuIxSI8q1P6BA84aWZdalyfDWBZ3FMjsiw==} + engines: {node: '>=16.0.0'} + '@drizzle-team/brocli@0.10.2': resolution: {integrity: sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w==} @@ -2039,6 +2055,10 @@ packages: resolution: {integrity: sha512-HcWLW28yTMGXpwE9VLx9J+N2KEUaELadLrkPEEI9tpI5la70xNEVEsu/C+m3u7uoq4FulLqZQhgBCzR9IZhFpA==} engines: {node: '>=20.0.0'} + '@sindresorhus/is@4.6.0': + resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==} + engines: {node: '>=10'} + '@sindresorhus/merge-streams@4.0.0': resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} engines: {node: '>=18'} @@ -2626,6 +2646,10 @@ packages: ansi-align@3.0.1: resolution: {integrity: sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==} + ansi-colors@4.1.3: + resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} + engines: {node: '>=6'} + ansi-escapes@4.3.2: resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==} engines: {node: '>=8'} @@ -2654,6 +2678,9 @@ packages: resolution: {integrity: sha512-0qWUglt9JEqLFr3w1I1pbrChn1grhaiAR2ocX1PP/flRmxgtwTzPFFFnfIlD6aMOLQZgSuCRlidD70lvx8yhzg==} engines: {node: '>=14'} + apify-client@2.20.0: + resolution: {integrity: sha512-oEMTImVVRZ5n8JkFV6dgbBFL3Xqz+GTwjUCjn/hwSNkow31Q8VNGk4qYDfRjkoqNQJ3ZirhtCwTnhkSXn1Tf+g==} + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -2672,6 +2699,9 @@ packages: resolution: {integrity: sha512-UOCGPYbl0tv8+006qks/dTgV9ajs97X2p0FAbyS2iyCRrmLSRolDaHdp+v/CLgnzHc3fVB+CwYiUmei7ndFcgA==} engines: {node: '>=12.0.0'} + async-retry@1.3.3: + resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} + async@3.2.6: resolution: {integrity: sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==} @@ -2860,6 +2890,10 @@ packages: resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} engines: {node: ^14.18.0 || >=16.10.0} + content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} @@ -2995,6 +3029,10 @@ packages: dompurify@3.1.7: resolution: {integrity: sha512-VaTstWtsneJY8xzy7DekmYWEOZcmzIe3Qb3zPd4STve1OBTa+e+WmS1ITQec1fZYXI3HCsOZZiSMpG6oxoWMWQ==} + dot-prop@6.0.1: + resolution: {integrity: sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==} + engines: {node: '>=10'} + dotenv@17.2.3: resolution: {integrity: sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==} engines: {node: '>=12'} @@ -3479,6 +3517,10 @@ packages: resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} engines: {node: '>=0.12.0'} + is-obj@2.0.0: + resolution: {integrity: sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==} + engines: {node: '>=8'} + is-plain-obj@4.1.0: resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} engines: {node: '>=12'} @@ -3684,6 +3726,10 @@ packages: resolution: {integrity: sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + lodash.isequal@4.5.0: + resolution: {integrity: sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==} + deprecated: This package is deprecated. Use require('node:util').isDeepStrictEqual instead. + log-symbols@6.0.0: resolution: {integrity: sha512-i24m8rpwhmPIS4zscNzK6MSEhk0DUWa/8iYQWxhffV8jkI4Phvs3F+quL5xvS0gdQR0FyTCMMH33Y78dDTzzIw==} engines: {node: '>=18'} @@ -3909,6 +3955,10 @@ packages: resolution: {integrity: sha512-wrAwOeXp1RRMFfQY8Sy7VaGVmPocaLwSFOYCGKSyo8qmJ+/yaafCl5BCA1IQZWqFSRBrKDYFeR9d/VyQzfH/jg==} engines: {node: '>= 6.0'} + ow@0.28.2: + resolution: {integrity: sha512-dD4UpyBh/9m4X2NVjA+73/ZPBRF+uF4zIMFvvQsabMiEK8x41L3rQ8EENOi35kyyoaJwNxEeJcP6Fj1H4U409Q==} + engines: {node: '>=12'} + oxc-resolver@11.13.2: resolution: {integrity: sha512-1SXVyYQ9bqMX3uZo8Px81EG7jhZkO9PvvR5X9roY5TLYVm4ZA7pbPDNlYaDBBeF9U+YO3OeMNoHde52hrcCu8w==} @@ -4522,6 +4572,10 @@ packages: resolution: {integrity: sha512-4MYBu2UuYq6wwNtqlOTUobeUYjXH+RzpSFRQKWOlRw18T47mjGq5Tp4odGS0GK7OGnUwxKG2Cm6JkLx6RLWmBA==} engines: {node: '>=22', pnpm: '>=9'} + vali-date@1.0.0: + resolution: {integrity: sha512-sgECfZthyaCKW10N0fm27cg8HYTFK5qMWgypqkXMQ4Wbl/zZKx7xZICgcoxIIE+WFAP/MBL2EFwC/YvLxw3Zeg==} + engines: {node: '>=0.10.0'} + vaul@1.1.2: resolution: {integrity: sha512-ZFkClGpWyI2WUQjdLJ/BaGuV6AVQiJ3uELGk3OYtP+B6yCO7Cmn9vPFXVJkRaGkOJu3m8bQMgtyzNHixULceQA==} peerDependencies: @@ -4674,6 +4728,18 @@ snapshots: '@alloc/quick-lru@5.2.0': {} + '@apify/consts@2.48.0': {} + + '@apify/log@2.5.28': + dependencies: + '@apify/consts': 2.48.0 + ansi-colors: 4.1.3 + + '@apify/utilities@2.24.0': + dependencies: + '@apify/consts': 2.48.0 + '@apify/log': 2.5.28 + '@aws-crypto/sha256-browser@5.2.0': dependencies: '@aws-crypto/sha256-js': 5.2.0 @@ -5204,6 +5270,10 @@ snapshots: picocolors: 1.1.1 sisteransi: 1.0.5 + '@crawlee/types@3.15.3': + dependencies: + tslib: 2.8.1 + '@drizzle-team/brocli@0.10.2': {} '@emnapi/core@1.7.1': @@ -6732,6 +6802,8 @@ snapshots: '@peculiar/asn1-x509': 2.5.0 '@peculiar/x509': 1.14.0 + '@sindresorhus/is@4.6.0': {} + '@sindresorhus/merge-streams@4.0.0': {} '@slack/logger@4.0.0': @@ -7487,6 +7559,8 @@ snapshots: dependencies: string-width: 4.2.3 + ansi-colors@4.1.3: {} + ansi-escapes@4.3.2: dependencies: type-fest: 0.21.3 @@ -7507,6 +7581,22 @@ snapshots: ansis@3.17.0: {} + apify-client@2.20.0: + dependencies: + '@apify/consts': 2.48.0 + '@apify/log': 2.5.28 + '@apify/utilities': 2.24.0 + '@crawlee/types': 3.15.3 + ansi-colors: 4.1.3 + async-retry: 1.3.3 + axios: 1.13.1 + content-type: 1.0.5 + ow: 0.28.2 + tslib: 2.8.1 + type-fest: 4.41.0 + transitivePeerDependencies: + - debug + argparse@2.0.1: {} aria-hidden@1.2.6: @@ -7523,6 +7613,10 @@ snapshots: pvutils: 1.1.5 tslib: 2.8.1 + async-retry@1.3.3: + dependencies: + retry: 0.13.1 + async@3.2.6: {} asynckit@0.4.0: {} @@ -7702,6 +7796,8 @@ snapshots: consola@3.4.2: {} + content-type@1.0.5: {} + convert-source-map@2.0.0: optional: true @@ -7816,6 +7912,10 @@ snapshots: dompurify@3.1.7: {} + dot-prop@6.0.1: + dependencies: + is-obj: 2.0.0 + dotenv@17.2.3: {} drizzle-kit@0.31.6: @@ -8224,6 +8324,8 @@ snapshots: is-number@7.0.0: {} + is-obj@2.0.0: {} + is-plain-obj@4.1.0: {} is-stream@2.0.1: {} @@ -8392,6 +8494,8 @@ snapshots: dependencies: p-locate: 6.0.0 + lodash.isequal@4.5.0: {} + log-symbols@6.0.0: dependencies: chalk: 5.6.2 @@ -8596,6 +8700,14 @@ snapshots: os-paths@4.4.0: {} + ow@0.28.2: + dependencies: + '@sindresorhus/is': 4.6.0 + callsites: 3.1.0 + dot-prop: 6.0.1 + lodash.isequal: 4.5.0 + vali-date: 1.0.0 + oxc-resolver@11.13.2: optionalDependencies: '@oxc-resolver/binding-android-arm-eabi': 11.13.2 @@ -9225,6 +9337,8 @@ snapshots: v0-sdk@0.15.1: {} + vali-date@1.0.0: {} + vaul@1.1.2(@types/react-dom@19.2.2(@types/react@19.2.2))(@types/react@19.2.2)(react-dom@19.2.1(react@19.2.1))(react@19.2.1): dependencies: '@radix-ui/react-dialog': 1.1.15(@types/react-dom@19.2.2(@types/react@19.2.2))(@types/react@19.2.2)(react-dom@19.2.1(react@19.2.1))(react@19.2.1) From 83a1f46f006248708dc325d57dbf894a1a3a631a Mon Sep 17 00:00:00 2001 From: drobnikj Date: Fri, 5 Dec 2025 15:17:09 +0100 Subject: [PATCH 18/18] fix: fix readme imports --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f28eda04..16a3d93d 100644 --- a/README.md +++ b/README.md @@ -272,10 +272,8 @@ const searchResult = await firecrawlSearchStep({ ### Apify (Web Scraping) ```typescript -import { - scrapeSingleUrlStep, - apifyRunActorStep, -} from "@/lib/steps/apify"; +import { apifyRunActorStep } from "@/lib/apify/steps/run-actor/step"; +import { scrapeSingleUrlStep } from "@/lib/apify/steps/scrape-single-url/step"; // Scrape a URL const scrapeResult = await scrapeSingleUrlStep({