fix LiteLLM model issue (#68)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Bug Fixes**
* LiteLLM provider no longer requires an API key (like BrowserOperator),
reducing spurious "API key not configured" errors.
* Model resolution now falls back intelligently (mini → main/model; nano
→ mini → main/model) and updates context to reflect actual resolved
models.
  * Error messages clarified for missing model fallbacks.

* **New Features**
* Schema extraction results can include optional metadata for downstream
use.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: tysonthomas9

front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts

@@ -460,9 +460,10 @@ export class ConfigurableAgentTool implements Tool<ConfigurableAgentArgs, Config
     const apiKey = callCtx.apiKey;
     const provider = callCtx.provider;
 
-    // Check if this provider requires an API key
-    // BrowserOperator doesn't require an API key (endpoint is hardcoded)
-    const requiresApiKey = provider !== 'browseroperator';
+    // Check if API key is required based on provider
+    // LiteLLM and BrowserOperator have optional API keys
+    // Other providers (OpenAI, Groq, OpenRouter) require API keys
+    const requiresApiKey = provider !== 'litellm' && provider !== 'browseroperator';
 
     if (requiresApiKey && !apiKey) {
       const errorResult = this.createErrorResult(`API key not configured for ${this.name}`, [], 'error');
@@ -499,15 +500,17 @@ export class ConfigurableAgentTool implements Tool<ConfigurableAgentArgs, Config
     // Resolve model name from context or configuration
     let modelName: string;
     if (this.config.modelName === MODEL_SENTINELS.USE_MINI) {
-      if (!callCtx.miniModel) {
-        throw new Error(`Mini model not provided in context for agent '${this.name}'. Ensure context includes miniModel.`);
+      // Fall back to main model if mini model is not configured
+      modelName = callCtx.miniModel || callCtx.mainModel || callCtx.model || '';
+      if (!modelName) {
+        throw new Error(`Mini model not provided in context for agent '${this.name}'. Ensure context includes miniModel or mainModel.`);
       }
-      modelName = callCtx.miniModel;
     } else if (this.config.modelName === MODEL_SENTINELS.USE_NANO) {
-      if (!callCtx.nanoModel) {
-        throw new Error(`Nano model not provided in context for agent '${this.name}'. Ensure context includes nanoModel.`);
+      // Fall back through nano -> mini -> main model chain
+      modelName = callCtx.nanoModel || callCtx.miniModel || callCtx.mainModel || callCtx.model || '';
+      if (!modelName) {
+        throw new Error(`Nano model not provided in context for agent '${this.name}'. Ensure context includes nanoModel, miniModel, or mainModel.`);
       }
-      modelName = callCtx.nanoModel;
     } else if (typeof this.config.modelName === 'function') {
       modelName = this.config.modelName();
     } else if (this.config.modelName) {
@@ -525,16 +528,25 @@ export class ConfigurableAgentTool implements Tool<ConfigurableAgentArgs, Config
     if (callCtx.model && !this.config.modelName) {
       modelName = callCtx.model;
     }
-    
+
+    // Update context with resolved fallback models for tools to use
+    // This ensures tools that check ctx.miniModel or ctx.nanoModel get the fallback
+    if (this.config.modelName === MODEL_SENTINELS.USE_MINI && !callCtx.miniModel) {
+      callCtx.miniModel = modelName;  // Use the resolved fallback
+    }
+    if (this.config.modelName === MODEL_SENTINELS.USE_NANO && !callCtx.nanoModel) {
+      callCtx.nanoModel = modelName;  // Use the resolved fallback
+    }
+
     // Validate required context
     if (!callCtx.provider) {
       throw new Error(`Provider not provided in context for agent '${this.name}'. Ensure context includes provider.`);
     }
-    
+
     const temperature = this.config.temperature ?? 0;
     const systemPrompt = this.config.systemPrompt;
     const tools = this.getToolInstances();
-    
+
     // Prepare initial messages
     const internalMessages = this.prepareInitialMessages(args);
     const runnerConfig: AgentRunnerConfig = {

front_end/panels/ai_chat/tools/CombinedExtractionTool.ts

@@ -87,8 +87,8 @@ export class CombinedExtractionTool implements Tool<CombinedExtractionArgs, Comb
     // Get provider from context
     const provider = ctx?.provider;
 
-    // BrowserOperator doesn't require API key
-    const requiresApiKey = provider !== 'browseroperator';
+    // LiteLLM and BrowserOperator have optional API keys
+    const requiresApiKey = provider !== 'litellm' && provider !== 'browseroperator';
 
     if (requiresApiKey && !apiKey) {
       return {

front_end/panels/ai_chat/tools/FullPageAccessibilityTreeToMarkdownTool.ts

@@ -55,8 +55,8 @@ export class FullPageAccessibilityTreeToMarkdownTool implements Tool<Record<stri
     const provider = ctx.provider;
     const model = ctx.nanoModel;
 
-    // BrowserOperator doesn't require API key
-    const requiresApiKey = provider !== 'browseroperator';
+    // LiteLLM and BrowserOperator have optional API keys
+    const requiresApiKey = provider !== 'litellm' && provider !== 'browseroperator';
 
     if (requiresApiKey && !apiKey) {
       return { error: 'API key not configured.' };

front_end/panels/ai_chat/tools/HTMLToMarkdownTool.ts

@@ -67,8 +67,8 @@ export class HTMLToMarkdownTool implements Tool<HTMLToMarkdownArgs, HTMLToMarkdo
     // Get provider from context
     const provider = ctx?.provider;
 
-    // BrowserOperator doesn't require API key
-    const requiresApiKey = provider !== 'browseroperator';
+    // LiteLLM and BrowserOperator have optional API keys
+    const requiresApiKey = provider !== 'litellm' && provider !== 'browseroperator';
 
     if (requiresApiKey && !apiKey) {
       return {

front_end/panels/ai_chat/tools/SchemaBasedExtractorTool.ts

@@ -108,8 +108,8 @@ Schema Examples:
     // Get provider from context
     const provider = ctx?.provider;
 
-    // BrowserOperator doesn't require API key
-    const requiresApiKey = provider !== 'browseroperator';
+    // LiteLLM and BrowserOperator have optional API keys
+    const requiresApiKey = provider !== 'litellm' && provider !== 'browseroperator';
 
     if (requiresApiKey && !apiKey) {
       return {

front_end/panels/ai_chat/tools/StreamlinedSchemaExtractorTool.ts

@@ -113,8 +113,8 @@ export class StreamlinedSchemaExtractorTool implements Tool<StreamlinedSchemaExt
     // Get provider from context
     const provider = ctx?.provider;
 
-    // BrowserOperator doesn't require API key
-    const requiresApiKey = provider !== 'browseroperator';
+    // LiteLLM and BrowserOperator have optional API keys
+    const requiresApiKey = provider !== 'litellm' && provider !== 'browseroperator';
 
     if (requiresApiKey && !apiKey) {
       return {

front_end/panels/ai_chat/tools/Tools.ts

@@ -2967,8 +2967,8 @@ Important guidelines:
       return { error: 'Missing LLM context (provider/model) for ObjectiveDrivenActionTool' };
     }
 
-    // BrowserOperator doesn't require API key
-    const requiresApiKey = providerForAction !== 'browseroperator';
+    // LiteLLM and BrowserOperator have optional API keys
+    const requiresApiKey = providerForAction !== 'litellm' && providerForAction !== 'browseroperator';
 
     if (requiresApiKey && !apiKey) {return { error: 'API key not configured.' };}
     if (typeof objective !== 'string' || objective.trim() === '') {