Skip to content

Commit a90e4d7

Browse files
tysonthomas9tysonthomas9
committed
More improvements and additional tools
1 parent 8a12eda commit a90e4d7

File tree

16 files changed

+1591
-87
lines changed

16 files changed

+1591
-87
lines changed

‎front_end/panels/ai_chat/BUILD.gn‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,11 +99,14 @@ devtools_module("ai_chat") {
9999
"tools/DeleteFileTool.ts",
100100
"tools/ReadFileTool.ts",
101101
"tools/ListFilesTool.ts",
102+
"tools/UpdateTodoTool.ts",
103+
"tools/ExecuteCodeTool.ts",
102104
"tools/SequentialThinkingTool.ts",
103105
"tools/ThinkingTool.ts",
104106
"tools/RenderWebAppTool.ts",
105107
"tools/GetWebAppDataTool.ts",
106108
"tools/RemoveWebAppTool.ts",
109+
"tools/VisualIndicatorTool.ts",
107110
"agent_framework/ConfigurableAgentTool.ts",
108111
"agent_framework/AgentRunner.ts",
109112
"agent_framework/AgentRunnerEventBus.ts",
@@ -267,11 +270,14 @@ _ai_chat_sources = [
267270
"tools/DeleteFileTool.ts",
268271
"tools/ReadFileTool.ts",
269272
"tools/ListFilesTool.ts",
273+
"tools/UpdateTodoTool.ts",
274+
"tools/ExecuteCodeTool.ts",
270275
"tools/SequentialThinkingTool.ts",
271276
"tools/ThinkingTool.ts",
272277
"tools/RenderWebAppTool.ts",
273278
"tools/GetWebAppDataTool.ts",
274279
"tools/RemoveWebAppTool.ts",
280+
"tools/VisualIndicatorTool.ts",
275281
"agent_framework/ConfigurableAgentTool.ts",
276282
"agent_framework/AgentRunner.ts",
277283
"agent_framework/AgentRunnerEventBus.ts",

‎front_end/panels/ai_chat/agent_framework/AgentRunner.ts‎

Lines changed: 109 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import { AgentErrorHandler } from '../core/AgentErrorHandler.js';
1515
import { AgentRunnerEventBus } from './AgentRunnerEventBus.js';
1616
import { callLLMWithTracing } from '../tools/LLMTracingWrapper.js';
1717
import { sanitizeMessagesForModel } from '../LLM/MessageSanitizer.js';
18+
import { FileStorageManager } from '../tools/FileStorageManager.js';
1819

1920
const logger = createLogger('AgentRunner');
2021

@@ -555,6 +556,24 @@ export class AgentRunner {
555556
// Check if execution has been aborted
556557
if (abortSignal?.aborted) {
557558
logger.info(`${agentName} execution aborted at iteration ${iteration + 1}/${maxIterations}`);
559+
560+
// Complete session with abort
561+
currentSession.status = 'error';
562+
currentSession.endTime = new Date();
563+
currentSession.terminationReason = 'error';
564+
565+
// Emit session completed event
566+
if (AgentRunner.eventBus) {
567+
AgentRunner.eventBus.emitProgress({
568+
type: 'session_completed',
569+
sessionId: currentSession.sessionId,
570+
parentSessionId: currentSession.parentSessionId,
571+
agentName,
572+
timestamp: new Date(),
573+
data: { session: currentSession, reason: 'aborted' }
574+
});
575+
}
576+
558577
const abortResult = createErrorResult('Execution was cancelled', messages, 'error');
559578
return { ...abortResult, agentSession: currentSession };
560579
}
@@ -571,9 +590,25 @@ export class AgentRunner {
571590
- You are currently on step ${iteration + 1} of ${maxIterations} maximum steps.
572591
- Focus on making meaningful progress with each step.`;
573592

574-
// Enhance system prompt with iteration info and page context
593+
// Inject todos into system prompt if they exist
594+
let todosContext = '';
595+
try {
596+
const fileManager = FileStorageManager.getInstance();
597+
const todosFile = await fileManager.readFile('todos.md');
598+
599+
if (todosFile?.content) {
600+
todosContext = `\n\n## CURRENT TODO LIST\n${todosFile.content}\n\nUpdate the todo list using the 'update_todo' tool as you complete tasks. Mark completed items with [x].`;
601+
} else {
602+
todosContext = `\n\n## TODO LIST\nNo todo list exists yet. If this is a multi-step task, create a todo list using the 'update_todo' tool to track your progress.`;
603+
}
604+
} catch (error) {
605+
logger.debug('Failed to read todos, skipping injection:', error);
606+
// Continue without todos if reading fails
607+
}
608+
609+
// Enhance system prompt with iteration info, todos, and page context
575610
// This includes updating the accessibility tree inside enhancePromptWithPageContext
576-
const currentSystemPrompt = await enhancePromptWithPageContext(systemPrompt + iterationInfo);
611+
const currentSystemPrompt = await enhancePromptWithPageContext(systemPrompt + iterationInfo + todosContext);
577612

578613
let llmResponse: LLMResponse;
579614
let generationId: string | undefined; // Declare in iteration scope for tool call access
@@ -753,6 +788,18 @@ export class AgentRunner {
753788
agentSession.endTime = new Date();
754789
agentSession.terminationReason = 'error';
755790

791+
// Emit session completed event
792+
if (AgentRunner.eventBus) {
793+
AgentRunner.eventBus.emitProgress({
794+
type: 'session_completed',
795+
sessionId: agentSession.sessionId,
796+
parentSessionId: agentSession.parentSessionId,
797+
agentName,
798+
timestamp: new Date(),
799+
data: { session: agentSession, reason: 'error' }
800+
});
801+
}
802+
756803
// Use error hook with structured summary
757804
const result = createErrorResult(errorMsg, messages, 'error');
758805
result.summary = {
@@ -909,6 +956,18 @@ export class AgentRunner {
909956
agentSession.endTime = new Date();
910957
agentSession.terminationReason = 'handed_off';
911958

959+
// Emit session completed event
960+
if (AgentRunner.eventBus) {
961+
AgentRunner.eventBus.emitProgress({
962+
type: 'session_completed',
963+
sessionId: agentSession.sessionId,
964+
parentSessionId: agentSession.parentSessionId,
965+
agentName,
966+
timestamp: new Date(),
967+
data: { session: agentSession, reason: 'handed_off' }
968+
});
969+
}
970+
912971
return { ...handoffResult, agentSession };
913972

914973
} else if (toolToExecute) {
@@ -1195,6 +1254,18 @@ export class AgentRunner {
11951254
agentSession.endTime = new Date();
11961255
agentSession.terminationReason = 'final_answer';
11971256

1257+
// Emit session completed event
1258+
if (AgentRunner.eventBus) {
1259+
AgentRunner.eventBus.emitProgress({
1260+
type: 'session_completed',
1261+
sessionId: agentSession.sessionId,
1262+
parentSessionId: agentSession.parentSessionId,
1263+
agentName,
1264+
timestamp: new Date(),
1265+
data: { session: agentSession, reason: 'final_answer' }
1266+
});
1267+
}
1268+
11981269
// Exit loop and return success with structured summary
11991270
const result = createSuccessResult(answer, messages, 'final_answer');
12001271
result.summary = {
@@ -1238,6 +1309,18 @@ export class AgentRunner {
12381309
agentSession.endTime = new Date();
12391310
agentSession.terminationReason = 'error';
12401311

1312+
// Emit session completed event
1313+
if (AgentRunner.eventBus) {
1314+
AgentRunner.eventBus.emitProgress({
1315+
type: 'session_completed',
1316+
sessionId: agentSession.sessionId,
1317+
parentSessionId: agentSession.parentSessionId,
1318+
agentName,
1319+
timestamp: new Date(),
1320+
data: { session: agentSession, reason: 'error' }
1321+
});
1322+
}
1323+
12411324
// Use error hook with structured summary
12421325
const result = createErrorResult(errorMsg, messages, 'error');
12431326
result.summary = {
@@ -1286,6 +1369,18 @@ export class AgentRunner {
12861369
agentSession.endTime = new Date();
12871370
agentSession.terminationReason = 'handed_off';
12881371

1372+
// Emit session completed event
1373+
if (AgentRunner.eventBus) {
1374+
AgentRunner.eventBus.emitProgress({
1375+
type: 'session_completed',
1376+
sessionId: agentSession.sessionId,
1377+
parentSessionId: agentSession.parentSessionId,
1378+
agentName,
1379+
timestamp: new Date(),
1380+
data: { session: agentSession, reason: 'handed_off' }
1381+
});
1382+
}
1383+
12891384
return { ...actualResult, agentSession }; // Return the result from the handoff target
12901385
}
12911386
}
@@ -1298,6 +1393,18 @@ export class AgentRunner {
12981393
agentSession.endTime = new Date();
12991394
agentSession.terminationReason = 'max_iterations';
13001395

1396+
// Emit session completed event
1397+
if (AgentRunner.eventBus) {
1398+
AgentRunner.eventBus.emitProgress({
1399+
type: 'session_completed',
1400+
sessionId: agentSession.sessionId,
1401+
parentSessionId: agentSession.parentSessionId,
1402+
agentName,
1403+
timestamp: new Date(),
1404+
data: { session: agentSession, reason: 'max_iterations' }
1405+
});
1406+
}
1407+
13011408
// Generate summary of agent progress instead of generic error message
13021409
const progressSummary = await this.summarizeAgentProgress(messages, maxIterations, agentName, modelName, 'max_iterations', config.provider, config.getVisionCapability);
13031410
const result = createErrorResult('Agent reached maximum iterations', messages, 'max_iterations');

‎front_end/panels/ai_chat/agent_framework/AgentRunnerEventBus.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import * as Common from '../../../core/common/common.js';
66

77
export interface AgentRunnerProgressEvent {
8-
type: 'session_started' | 'tool_started' | 'tool_completed' | 'session_updated' | 'child_agent_started';
8+
type: 'session_started' | 'tool_started' | 'tool_completed' | 'session_updated' | 'child_agent_started' | 'session_completed';
99
sessionId: string;
1010
parentSessionId?: string;
1111
agentName: string;

‎front_end/panels/ai_chat/agent_framework/implementation/ConfiguredAgents.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import { StreamlinedSchemaExtractorTool } from '../../tools/StreamlinedSchemaExt
99
import { BookmarkStoreTool } from '../../tools/BookmarkStoreTool.js';
1010
import { DocumentSearchTool } from '../../tools/DocumentSearchTool.js';
1111
import { NavigateURLTool, PerformActionTool, GetAccessibilityTreeTool, SearchContentTool, NavigateBackTool, NodeIDsToURLsTool, TakeScreenshotTool, ScrollPageTool, WaitTool, RenderWebAppTool, GetWebAppDataTool, RemoveWebAppTool, CreateFileTool, UpdateFileTool, DeleteFileTool, ReadFileTool, ListFilesTool } from '../../tools/Tools.js';
12+
import { UpdateTodoTool } from '../../tools/UpdateTodoTool.js';
13+
import { ExecuteCodeTool } from '../../tools/ExecuteCodeTool.js';
1214
import { HTMLToMarkdownTool } from '../../tools/HTMLToMarkdownTool.js';
1315
import { ConfigurableAgentTool, ToolRegistry } from '../ConfigurableAgentTool.js';
1416
import { ThinkingTool } from '../../tools/ThinkingTool.js';
@@ -54,6 +56,8 @@ export function initializeConfiguredAgents(): void {
5456
ToolRegistry.registerToolFactory('delete_file', () => new DeleteFileTool());
5557
ToolRegistry.registerToolFactory('read_file', () => new ReadFileTool());
5658
ToolRegistry.registerToolFactory('list_files', () => new ListFilesTool());
59+
ToolRegistry.registerToolFactory('update_todo', () => new UpdateTodoTool());
60+
ToolRegistry.registerToolFactory('execute_code', () => new ExecuteCodeTool());
5761

5862
// Register webapp rendering tools
5963
ToolRegistry.registerToolFactory('render_webapp', () => new RenderWebAppTool());

‎front_end/panels/ai_chat/agent_framework/implementation/agents/ResearchAgent.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@ Before handing off, ensure your latest findings are reflected in the shared file
158158
'navigate_back',
159159
'fetcher_tool',
160160
'extract_data',
161+
'execute_code',
161162
'node_ids_to_urls',
162163
'bookmark_store',
163164
'document_search',
@@ -167,6 +168,7 @@ Before handing off, ensure your latest findings are reflected in the shared file
167168
'delete_file',
168169
'read_file',
169170
'list_files',
171+
'update_todo',
170172
],
171173
maxIterations: 15,
172174
modelName: MODEL_SENTINELS.USE_MINI,

‎front_end/panels/ai_chat/agent_framework/implementation/agents/SearchAgent.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ If you absolutely cannot find any reliable leads, return status "failed" with ga
130130
'node_ids_to_urls',
131131
'fetcher_tool',
132132
'extract_data',
133+
'execute_code',
133134
'scroll_page',
134135
'action_agent',
135136
'html_to_markdown',
@@ -138,6 +139,7 @@ If you absolutely cannot find any reliable leads, return status "failed" with ga
138139
'delete_file',
139140
'read_file',
140141
'list_files',
142+
'update_todo',
141143
],
142144
maxIterations: 12,
143145
modelName: MODEL_SENTINELS.USE_MINI,

‎front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ Remember: **Plan adaptively, execute systematically, validate continuously, and
198198
'navigate_back',
199199
'action_agent',
200200
'extract_data',
201+
'execute_code',
201202
'node_ids_to_urls',
202203
'direct_url_navigator_agent',
203204
'scroll_page',
@@ -212,6 +213,7 @@ Remember: **Plan adaptively, execute systematically, validate continuously, and
212213
'delete_file',
213214
'read_file',
214215
'list_files',
216+
'update_todo',
215217
],
216218
maxIterations: 15,
217219
temperature: 0.3,

‎front_end/panels/ai_chat/core/AgentNodes.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -891,15 +891,19 @@ export function createToolExecutorNode(state: AgentState, provider: LLMProvider,
891891
}
892892

893893
// Special handling for ConfigurableAgentTool results
894-
if (selectedTool instanceof ConfigurableAgentTool && result && typeof result === 'object' &&
894+
if (selectedTool instanceof ConfigurableAgentTool && result && typeof result === 'object' &&
895895
('output' in result || 'error' in result || 'success' in result)) {
896896
// For ConfigurableAgentTool, only send the output/error fields to the LLM, never intermediateSteps
897897
const agentResult = result as any; // Cast to any to access ConfigurableAgentResult properties
898-
resultText = agentResult.output || (agentResult.error ? `Error: ${agentResult.error}` : 'No output');
898+
// Prioritize summary.content (detailed LLM analysis), fallback to output/error
899+
resultText = agentResult.summary?.content
900+
|| agentResult.output
901+
|| (agentResult.error ? `Error: ${agentResult.error}` : 'No output');
899902
logger.debug(`Filtered ConfigurableAgentTool result for LLM:`, {
900903
toolName,
901904
originalResult: result,
902-
filteredResult: resultText
905+
filteredResult: resultText,
906+
hasSummary: !!agentResult.summary?.content
903907
});
904908
} else if (toolName === 'finalize_with_critique') {
905909
logger.debug('ToolExecutorNode: finalize_with_critique result:', result);

‎front_end/panels/ai_chat/core/AgentService.ts‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import { AgentRunner } from '../agent_framework/AgentRunner.js';
2323
import type { AgentSession, AgentMessage } from '../agent_framework/AgentSessionTypes.js';
2424
import type { LLMProvider } from '../LLM/LLMTypes.js';
2525
import { BUILD_CONFIG } from './BuildConfig.js';
26+
import { VisualIndicatorManager } from '../tools/VisualIndicatorTool.js';
2627

2728
// Cache break: 2025-09-17T17:54:00Z - Force rebuild with AUTOMATED_MODE bypass
2829
const logger = createLogger('AgentService');
@@ -36,6 +37,7 @@ export enum Events {
3637
AGENT_TOOL_STARTED = 'agent-tool-started',
3738
AGENT_TOOL_COMPLETED = 'agent-tool-completed',
3839
AGENT_SESSION_UPDATED = 'agent-session-updated',
40+
AGENT_SESSION_COMPLETED = 'agent-session-completed',
3941
CHILD_AGENT_STARTED = 'child-agent-started',
4042
}
4143

@@ -48,6 +50,7 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{
4850
[Events.AGENT_TOOL_STARTED]: { session: AgentSession, toolCall: AgentMessage },
4951
[Events.AGENT_TOOL_COMPLETED]: { session: AgentSession, toolResult: AgentMessage },
5052
[Events.AGENT_SESSION_UPDATED]: AgentSession,
53+
[Events.AGENT_SESSION_COMPLETED]: AgentSession,
5154
[Events.CHILD_AGENT_STARTED]: { parentSession: AgentSession, childAgentName: string, childSessionId: string },
5255
}> {
5356
static instance: AgentService;
@@ -120,10 +123,13 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{
120123

121124
// Initialize AgentRunner event system
122125
AgentRunner.initializeEventBus();
123-
126+
124127
// Subscribe to AgentRunner events
125128
AgentRunnerEventBus.getInstance().addEventListener('agent-progress', this.#handleAgentProgress.bind(this));
126129

130+
// Initialize visual indicator system
131+
VisualIndicatorManager.getInstance().initialize();
132+
127133
// Subscribe to configuration changes
128134
this.#configManager.addChangeListener(this.#handleConfigurationChange.bind(this));
129135
}
@@ -857,6 +863,39 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{
857863
}
858864
}
859865
break;
866+
case 'session_completed':
867+
// Get the completed session from the event data or active sessions
868+
const completedSession = progressEvent.data?.session ||
869+
this.#activeAgentSessions.get(progressEvent.sessionId);
870+
871+
if (completedSession) {
872+
logger.info('[AgentService] Session completed:', {
873+
sessionId: progressEvent.sessionId,
874+
status: completedSession.status,
875+
terminationReason: completedSession.terminationReason
876+
});
877+
878+
// Update the session in our tracking with the completed state
879+
this.#activeAgentSessions.set(progressEvent.sessionId, completedSession);
880+
881+
// Upsert the completed session to messages (shows final_answer in transcript)
882+
this.#upsertAgentSessionInMessages(completedSession);
883+
884+
// Dispatch completion event for UI components
885+
this.dispatchEventToListeners(Events.AGENT_SESSION_COMPLETED, completedSession);
886+
887+
// Also dispatch session updated for components listening to that
888+
this.dispatchEventToListeners(Events.AGENT_SESSION_UPDATED, completedSession);
889+
890+
// Trigger messages changed to update the chat transcript
891+
this.dispatchEventToListeners(Events.MESSAGES_CHANGED, [...this.#state.messages]);
892+
893+
// Clean up after a short delay (5 seconds) to allow UI to finish rendering
894+
this.#cleanupCompletedSession(progressEvent.sessionId);
895+
} else {
896+
logger.warn('[AgentService] Session completed but session not found:', progressEvent.sessionId);
897+
}
898+
break;
860899
}
861900
}
862901

0 commit comments

Comments
 (0)