diff --git a/README.md b/README.md index d6a5e29e..3abf96aa 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ Visit [http://localhost:3000](http://localhost:3000) to get started. - **Firecrawl**: Scrape URL, Search Web - **GitHub**: Create Issue, List Issues, Get Issue, Update Issue - **Linear**: Create Ticket, Find Issues +- **Olostep**: Scrape URL, Search Web, Map Website, AI Answer - **Perplexity**: Search Web, Ask Question, Research Topic - **Resend**: Send Email - **Slack**: Send Slack Message diff --git a/plugins/index.ts b/plugins/index.ts index c2b41249..23acca02 100644 --- a/plugins/index.ts +++ b/plugins/index.ts @@ -20,6 +20,7 @@ import "./fal"; import "./firecrawl"; import "./github"; import "./linear"; +import "./olostep"; import "./perplexity"; import "./resend"; import "./slack"; diff --git a/plugins/olostep/credentials.ts b/plugins/olostep/credentials.ts new file mode 100644 index 00000000..376ce334 --- /dev/null +++ b/plugins/olostep/credentials.ts @@ -0,0 +1,4 @@ +export type OlostepCredentials = { + OLOSTEP_API_KEY?: string; +}; + diff --git a/plugins/olostep/icon.tsx b/plugins/olostep/icon.tsx new file mode 100644 index 00000000..b77bca01 --- /dev/null +++ b/plugins/olostep/icon.tsx @@ -0,0 +1,24 @@ +export function OlostepIcon({ className }: { className?: string }) { + return ( + + Olostep + + + O + + + ); +} diff --git a/plugins/olostep/index.ts b/plugins/olostep/index.ts new file mode 100644 index 00000000..366c1ffb --- /dev/null +++ b/plugins/olostep/index.ts @@ -0,0 +1,168 @@ +import type { IntegrationPlugin } from "../registry"; +import { registerIntegration } from "../registry"; +import { OlostepIcon } from "./icon"; + +const olostepPlugin: IntegrationPlugin = { + type: "olostep", + label: "Olostep", + description: "Web Data API for AI - Search, extract, and structure web data", + + icon: OlostepIcon, + + formFields: [ + { + id: "olostepApiKey", + label: "API Key", + type: "password", + placeholder: "ols_...", + configKey: "olostepApiKey", + envVar: "OLOSTEP_API_KEY", + helpText: "Get your API key from ", + helpLink: { + text: "olostep.com", + url: "https://olostep.com/dashboard", + }, + }, + ], + + testConfig: { + getTestFunction: async () => { + const { testOlostep } = await import("./test"); + return testOlostep; + }, + }, + + actions: [ + { + slug: "scrape", + label: "Scrape URL", + description: "Extract content from any URL with full JavaScript rendering", + category: "Olostep", + stepFunction: "olostepScrapeStep", + stepImportPath: "scrape", + outputFields: [ + { field: "markdown", description: "Scraped content as markdown" }, + { field: "html", description: "Raw HTML content" }, + { field: "metadata", description: "Page metadata object" }, + ], + configFields: [ + { + key: "url", + label: "URL", + type: "template-input", + placeholder: "https://example.com or {{NodeName.url}}", + example: "https://example.com", + required: true, + }, + { + key: "waitForSelector", + label: "Wait for Selector", + type: "text", + placeholder: "CSS selector to wait for (optional)", + example: ".main-content", + }, + ], + }, + { + slug: "search", + label: "Search Web", + description: "Search the web using Google Search via Olostep", + category: "Olostep", + stepFunction: "olostepSearchStep", + stepImportPath: "search", + outputFields: [ + { field: "results", description: "Array of search results" }, + { field: "totalResults", description: "Total number of results" }, + ], + configFields: [ + { + key: "query", + label: "Search Query", + type: "template-input", + placeholder: "Search query or {{NodeName.query}}", + example: "latest AI news", + required: true, + }, + { + key: "limit", + label: "Result Limit", + type: "number", + placeholder: "10", + min: 1, + example: "10", + }, + { + key: "country", + label: "Country", + type: "text", + placeholder: "Country code (e.g., us, uk)", + example: "us", + }, + ], + }, + { + slug: "map", + label: "Map Website", + description: "Discover all URLs from a website (sitemap discovery)", + category: "Olostep", + stepFunction: "olostepMapStep", + stepImportPath: "map", + outputFields: [ + { field: "urls", description: "Array of discovered URLs" }, + { field: "count", description: "Number of URLs found" }, + ], + configFields: [ + { + key: "url", + label: "Website URL", + type: "template-input", + placeholder: "https://example.com or {{NodeName.url}}", + example: "https://example.com", + required: true, + }, + { + key: "limit", + label: "Max URLs", + type: "number", + placeholder: "100", + min: 1, + example: "100", + }, + ], + }, + { + slug: "answer", + label: "AI Answer", + description: "Get AI-powered answers from web content", + category: "Olostep", + stepFunction: "olostepAnswerStep", + stepImportPath: "answer", + outputFields: [ + { field: "answer", description: "AI-generated answer" }, + { field: "sources", description: "Array of source URLs used" }, + ], + configFields: [ + { + key: "question", + label: "Question", + type: "template-input", + placeholder: "What is the latest news about...?", + example: "What are the key features of this product?", + required: true, + }, + { + key: "searchQuery", + label: "Search Query (optional)", + type: "template-input", + placeholder: "Search the web first for context", + example: "product features comparison 2024", + }, + ], + }, + ], +}; + +// Auto-register on import +registerIntegration(olostepPlugin); + +export default olostepPlugin; diff --git a/plugins/olostep/steps/answer.ts b/plugins/olostep/steps/answer.ts new file mode 100644 index 00000000..b5b726c1 --- /dev/null +++ b/plugins/olostep/steps/answer.ts @@ -0,0 +1,95 @@ +import "server-only"; + +import { fetchCredentials } from "@/lib/credential-fetcher"; +import { type StepInput, withStepLogging } from "@/lib/steps/step-handler"; +import { getErrorMessage } from "@/lib/utils"; + +type AnswerResult = { + answer: string; + sources: Array<{ + url: string; + title?: string; + }>; +}; + +export type OlostepAnswerInput = StepInput & { + integrationId?: string; + question: string; + urls?: string[]; + searchQuery?: string; +}; + +/** + * Answer logic using Olostep API + * Get AI-powered answers from web content + */ +async function getAnswer(input: OlostepAnswerInput): Promise { + const credentials = input.integrationId + ? await fetchCredentials(input.integrationId) + : {}; + + const apiKey = credentials.OLOSTEP_API_KEY; + + if (!apiKey) { + throw new Error("Olostep API Key is not configured."); + } + + try { + const requestBody: Record = { + question: input.question, + }; + + // If URLs are provided, use them as context + if (input.urls && input.urls.length > 0) { + requestBody.urls = input.urls; + } + + // If search query is provided, search first + if (input.searchQuery) { + requestBody.search_query = input.searchQuery; + } + + const response = await fetch("https://api.olostep.com/v1/answer", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify(requestBody), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Olostep API error: ${errorText}`); + } + + const result = await response.json(); + + return { + answer: result.answer || result.response || "", + sources: (result.sources || result.references || []) + .map((source: { url?: string; link?: string; title?: string }) => ({ + url: source.url || source.link || "", + title: source.title, + })) + .filter((source: { url: string; title?: string }) => source.url), + }; + } catch (error) { + throw new Error(`Failed to get answer: ${getErrorMessage(error)}`); + } +} + +/** + * Olostep Answer Step + * Gets AI-powered answers from web content using Olostep + */ +export async function olostepAnswerStep( + input: OlostepAnswerInput +): Promise { + "use step"; + return withStepLogging(input, () => getAnswer(input)); +} +olostepAnswerStep.maxRetries = 0; + +// Required for codegen auto-generation +export const _integrationType = "olostep"; diff --git a/plugins/olostep/steps/map.ts b/plugins/olostep/steps/map.ts new file mode 100644 index 00000000..38253147 --- /dev/null +++ b/plugins/olostep/steps/map.ts @@ -0,0 +1,80 @@ +import "server-only"; + +import { fetchCredentials } from "@/lib/credential-fetcher"; +import { type StepInput, withStepLogging } from "@/lib/steps/step-handler"; +import { getErrorMessage } from "@/lib/utils"; + +type MapResult = { + urls: string[]; + count: number; +}; + +export type OlostepMapInput = StepInput & { + integrationId?: string; + url: string; + limit?: number; + includeSubdomains?: boolean; +}; + +/** + * Map logic using Olostep API + * Gets all URLs from a website (sitemap discovery) + */ +async function mapUrls(input: OlostepMapInput): Promise { + const credentials = input.integrationId + ? await fetchCredentials(input.integrationId) + : {}; + + const apiKey = credentials.OLOSTEP_API_KEY; + + if (!apiKey) { + throw new Error("Olostep API Key is not configured."); + } + + try { + const response = await fetch("https://api.olostep.com/v1/map", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + url: input.url, + limit: input.limit || 100, + include_subdomains: input.includeSubdomains || false, + }), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Olostep API error: ${errorText}`); + } + + const result = await response.json(); + + const urls = result.urls || result.links || []; + const limitedUrls = urls.slice(0, input.limit || 100); + + return { + urls: limitedUrls, + count: limitedUrls.length, + }; + } catch (error) { + throw new Error(`Failed to map URLs: ${getErrorMessage(error)}`); + } +} + +/** + * Olostep Map Step + * Discovers all URLs from a website using Olostep + */ +export async function olostepMapStep( + input: OlostepMapInput +): Promise { + "use step"; + return withStepLogging(input, () => mapUrls(input)); +} +olostepMapStep.maxRetries = 0; + +// Required for codegen auto-generation +export const _integrationType = "olostep"; diff --git a/plugins/olostep/steps/scrape.ts b/plugins/olostep/steps/scrape.ts new file mode 100644 index 00000000..49dbc0fe --- /dev/null +++ b/plugins/olostep/steps/scrape.ts @@ -0,0 +1,82 @@ +import "server-only"; + +import { fetchCredentials } from "@/lib/credential-fetcher"; +import { type StepInput, withStepLogging } from "@/lib/steps/step-handler"; +import { getErrorMessage } from "@/lib/utils"; + +type ScrapeResult = { + markdown?: string; + html?: string; + metadata?: Record; +}; + +export type OlostepScrapeInput = StepInput & { + integrationId?: string; + url: string; + formats?: ("markdown" | "html" | "text")[]; + waitForSelector?: string; +}; + +/** + * Scrape logic using Olostep API + */ +async function scrape(input: OlostepScrapeInput): Promise { + const credentials = input.integrationId + ? await fetchCredentials(input.integrationId) + : {}; + + const apiKey = credentials.OLOSTEP_API_KEY; + + if (!apiKey) { + throw new Error("Olostep API Key is not configured."); + } + + try { + const response = await fetch("https://api.olostep.com/v1/scrapes", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + url_to_scrape: input.url, + formats: input.formats || ["markdown"], + wait_for_selector: input.waitForSelector, + }), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Olostep API error: ${errorText}`); + } + + const result = await response.json(); + + return { + markdown: result.markdown_content || result.markdown, + html: result.html_content || result.html, + metadata: { + title: result.title, + url: result.url, + statusCode: result.status_code, + }, + }; + } catch (error) { + throw new Error(`Failed to scrape: ${getErrorMessage(error)}`); + } +} + +/** + * Olostep Scrape Step + * Scrapes content from a URL using Olostep + */ +export async function olostepScrapeStep( + input: OlostepScrapeInput +): Promise { + "use step"; + return withStepLogging(input, () => scrape(input)); +} +olostepScrapeStep.maxRetries = 0; + +// Required for codegen auto-generation +export const _integrationType = "olostep"; diff --git a/plugins/olostep/steps/search.ts b/plugins/olostep/steps/search.ts new file mode 100644 index 00000000..b1746519 --- /dev/null +++ b/plugins/olostep/steps/search.ts @@ -0,0 +1,111 @@ +import "server-only"; + +import { fetchCredentials } from "@/lib/credential-fetcher"; +import { type StepInput, withStepLogging } from "@/lib/steps/step-handler"; +import { getErrorMessage } from "@/lib/utils"; + +type SearchResultItem = { + url: string; + title?: string; + description?: string; + markdown?: string; +}; + +type SearchResult = { + results: SearchResultItem[]; + totalResults: number; +}; + +export type OlostepSearchInput = StepInput & { + integrationId?: string; + query: string; + limit?: number; + country?: string; +}; + +/** + * Search logic using Olostep Google Search API + */ +async function search(input: OlostepSearchInput): Promise { + const credentials = input.integrationId + ? await fetchCredentials(input.integrationId) + : {}; + + const apiKey = credentials.OLOSTEP_API_KEY; + + if (!apiKey) { + throw new Error("Olostep API Key is not configured."); + } + + try { + // Use the map endpoint with google search for web search functionality + const params = new URLSearchParams({ + query: input.query, + limit: String(input.limit || 10), + }); + + if (input.country) { + params.append("country", input.country); + } + + const response = await fetch( + `https://api.olostep.com/v1/google-search?${params.toString()}`, + { + method: "GET", + headers: { + Authorization: `Bearer ${apiKey}`, + }, + } + ); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Olostep API error: ${errorText}`); + } + + const result = await response.json(); + + // Transform the response to a consistent format + // Filter before slicing to ensure we return the requested number of valid results + const results: SearchResultItem[] = (result.results || result.items || []) + .map( + (item: { + url?: string; + link?: string; + title?: string; + description?: string; + snippet?: string; + markdown?: string; + }) => ({ + url: item.url || item.link || "", + title: item.title, + description: item.description || item.snippet, + markdown: item.markdown, + }) + ) + .filter((item: SearchResultItem) => item.url) + .slice(0, input.limit || 10); + + return { + results, + totalResults: result.total_results || results.length, + }; + } catch (error) { + throw new Error(`Failed to search: ${getErrorMessage(error)}`); + } +} + +/** + * Olostep Search Step + * Searches the web using Olostep + */ +export async function olostepSearchStep( + input: OlostepSearchInput +): Promise { + "use step"; + return withStepLogging(input, () => search(input)); +} +olostepSearchStep.maxRetries = 0; + +// Required for codegen auto-generation +export const _integrationType = "olostep"; diff --git a/plugins/olostep/test.ts b/plugins/olostep/test.ts new file mode 100644 index 00000000..05497f89 --- /dev/null +++ b/plugins/olostep/test.ts @@ -0,0 +1,40 @@ +export async function testOlostep(credentials: Record) { + try { + const apiKey = credentials.OLOSTEP_API_KEY; + + if (!apiKey) { + return { + success: false, + error: "OLOSTEP_API_KEY is required", + }; + } + + const response = await fetch("https://api.olostep.com/v1/scrapes", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + url_to_scrape: "https://example.com", + formats: ["markdown"], + }), + }); + + if (response.ok) { + return { success: true }; + } + const error = await response.text(); + return { success: false, error }; + } catch (error) { + return { + success: false, + error: error instanceof Error ? error.message : String(error), + }; + } +} + + + + +