feat: add screen reader announcements for better accessibility

vick08 · vick08 · commit cc5c9c8b4fb7 · 2025-09-06T17:11:00.000-07:00
Implements comprehensive screen reader support with live region announcements for Claude's responses and tool execution. Includes message content cleaning to prevent duplicated announcements for screen readers.
diff --git a/src/components/ClaudeCodeSession.tsx b/src/components/ClaudeCodeSession.tsx
@@ -28,7 +28,7 @@ import { SplitPane } from "@/components/ui/split-pane";
 import { WebviewPreview } from "./WebviewPreview";
 import type { ClaudeStreamMessage } from "./AgentExecution";
 import { useVirtualizer } from "@tanstack/react-virtual";
-import { useTrackEvent, useComponentMetrics, useWorkflowTracking } from "@/hooks";
+import { useTrackEvent, useComponentMetrics, useWorkflowTracking, useScreenReaderAnnouncements } from "@/hooks";
 import { SessionPersistenceService } from "@/services/sessionPersistence";
 
 interface ClaudeCodeSessionProps {
@@ -139,6 +139,15 @@ export const ClaudeCodeSession: React.FC<ClaudeCodeSessionProps> = ({
   // const aiTracking = useAIInteractionTracking('sonnet'); // Default model
   const workflowTracking = useWorkflowTracking('claude_session');
   
+  // Screen reader announcements
+  const {
+    announceClaudeStarted,
+    announceClaudeFinished,
+    announceAssistantMessage,
+    announceToolExecution,
+    announceToolCompleted
+  } = useScreenReaderAnnouncements();
+  
   // Call onProjectPathChange when component mounts with initial path
   useEffect(() => {
     if (onProjectPathChange && projectPath) {
@@ -454,6 +463,9 @@ export const ClaudeCodeSession: React.FC<ClaudeCodeSessionProps> = ({
       setError(null);
       hasActiveSessionRef.current = true;
       
+      // Announce that Claude is starting to process
+      announceClaudeStarted();
+      
       // For resuming sessions, ensure we have the session ID
       if (effectiveSession && !claudeSessionId) {
         setClaudeSessionId(effectiveSession.id);
@@ -544,6 +556,60 @@ export const ClaudeCodeSession: React.FC<ClaudeCodeSessionProps> = ({
           }
         });
 
+        // Helper to find tool name by tool use ID from previous messages
+        function findToolNameById(toolUseId: string): string | null {
+          // Search backwards through messages for the tool use
+          for (let i = messages.length - 1; i >= 0; i--) {
+            const msg = messages[i];
+            if (msg.type === 'assistant' && msg.message?.content) {
+              const toolUse = msg.message.content.find((c: any) => 
+                c.type === 'tool_use' && c.id === toolUseId
+              );
+              if (toolUse?.name) {
+                return toolUse.name;
+              }
+            }
+          }
+          return null;
+        }
+
+        // Helper to announce incoming messages to screen readers
+        function announceIncomingMessage(message: ClaudeStreamMessage) {
+          if (message.type === 'assistant' && message.message?.content) {
+            // Announce tool execution
+            const toolUses = message.message.content.filter((c: any) => c.type === 'tool_use');
+            toolUses.forEach((toolUse: any) => {
+              const toolName = toolUse.name || 'unknown tool';
+              const description = toolUse.input?.description || 
+                                toolUse.input?.command || 
+                                toolUse.input?.file_path ||
+                                toolUse.input?.pattern ||
+                                toolUse.input?.prompt?.substring(0, 50);
+              announceToolExecution(toolName, description);
+            });
+            
+            // Announce text content
+            const textContent = message.message.content
+              .filter((c: any) => c.type === 'text')
+              .map((c: any) => typeof c.text === 'string' ? c.text : (c.text?.text || ''))
+              .join(' ')
+              .trim();
+              
+            if (textContent) {
+              announceAssistantMessage(textContent);
+            }
+          } else if (message.type === 'system') {
+            // Announce system messages if they have meaningful content
+            if (message.subtype === 'init') {
+              // Don't announce init messages as they're just setup
+              return;
+            } else if (message.result || message.error) {
+              const content = message.result || message.error || 'System message received';
+              announceAssistantMessage(content);
+            }
+          }
+        }
+
         // Helper to process any JSONL stream message string
         function handleStreamMessage(payload: string) {
           try {
@@ -555,6 +621,9 @@ export const ClaudeCodeSession: React.FC<ClaudeCodeSessionProps> = ({
 
             const message = JSON.parse(payload) as ClaudeStreamMessage;
             
+            // Announce incoming messages to screen readers
+            announceIncomingMessage(message);
+            
             // Track enhanced tool execution
             if (message.type === 'assistant' && message.message?.content) {
               const toolUses = message.message.content.filter((c: any) => c.type === 'tool_use');
@@ -583,6 +652,14 @@ export const ClaudeCodeSession: React.FC<ClaudeCodeSessionProps> = ({
               const toolResults = message.message.content.filter((c: any) => c.type === 'tool_result');
               toolResults.forEach((result: any) => {
                 const isError = result.is_error || false;
+                
+                // Announce tool completion
+                if (result.tool_use_id) {
+                  // Try to find the tool name from previous messages
+                  const toolName = findToolNameById(result.tool_use_id) || 'Tool';
+                  // announceToolCompleted(toolName, !isError); // Disabled to prevent interrupting other announcements
+                }
+                
                 // Note: We don't have execution time here, but we can track success/failure
                 if (isError) {
                   sessionMetrics.current.toolsFailed += 1;
@@ -634,6 +711,9 @@ export const ClaudeCodeSession: React.FC<ClaudeCodeSessionProps> = ({
           hasActiveSessionRef.current = false;
           isListeningRef.current = false; // Reset listening state
           
+          // Announce that Claude has finished
+          announceClaudeFinished();
+          
           // Track enhanced session stopped metrics when session completes
           if (effectiveSession && claudeSessionId) {
             const sessionStartTimeValue = messages.length > 0 ? messages[0].timestamp || Date.now() : Date.now();
diff --git a/src/components/StreamMessage.tsx b/src/components/StreamMessage.tsx
@@ -118,6 +118,7 @@ const StreamMessageComponent: React.FC<StreamMessageProps> = ({ message, classNa
             <div className="flex items-start gap-3">
               <Bot className="h-5 w-5 text-primary mt-0.5" />
               <div className="flex-1 space-y-2 min-w-0">
+                <h2 className="sr-only">Assistant Response</h2>
                 {msg.content && Array.isArray(msg.content) && msg.content.map((content: any, idx: number) => {
                   // Text content - render as markdown
                   if (content.type === "text") {
@@ -331,6 +332,7 @@ const StreamMessageComponent: React.FC<StreamMessageProps> = ({ message, classNa
             <div className="flex items-start gap-3">
               <User className="h-5 w-5 text-muted-foreground mt-0.5" />
               <div className="flex-1 space-y-2 min-w-0">
+                <h2 className="sr-only">User Prompt</h2>
                 {/* Handle content that is a simple string (e.g. from user commands) */}
                 {(typeof msg.content === 'string' || (msg.content && !Array.isArray(msg.content))) && (
                   (() => {
diff --git a/src/hooks/index.ts b/src/hooks/index.ts
@@ -24,3 +24,4 @@ export {
   useAsyncPerformanceTracker 
 } from './usePerformanceMonitor';
 export { TAB_SCREEN_NAMES } from './useAnalytics';
+export { useScreenReaderAnnouncements } from './useScreenReaderAnnouncements';
diff --git a/src/hooks/useScreenReaderAnnouncements.ts b/src/hooks/useScreenReaderAnnouncements.ts
@@ -0,0 +1,152 @@
+import { useCallback, useRef } from 'react';
+
+export interface ScreenReaderAnnouncementOptions {
+  priority?: 'polite' | 'assertive';
+  delay?: number;
+}
+
+/**
+ * Custom hook to manage screen reader announcements for incoming messages
+ * Uses ARIA live regions to announce content to screen readers
+ */
+export const useScreenReaderAnnouncements = () => {
+  const announcementTimeoutRef = useRef<NodeJS.Timeout | null>(null);
+  const lastAnnouncementRef = useRef<string>('');
+
+  /**
+   * Announces a message to screen readers via ARIA live regions
+   * @param message - The message content to announce
+   * @param options - Configuration options for the announcement
+   */
+  const announceMessage = useCallback((message: string, options: ScreenReaderAnnouncementOptions = {}) => {
+    const { priority = 'polite', delay = 100 } = options;
+    
+    // Clear any pending announcements
+    if (announcementTimeoutRef.current) {
+      clearTimeout(announcementTimeoutRef.current);
+    }
+    
+    // Don't announce the same message twice in a row
+    if (message === lastAnnouncementRef.current) {
+      return;
+    }
+    
+    lastAnnouncementRef.current = message;
+    
+    // Small delay to ensure DOM updates are complete
+    announcementTimeoutRef.current = setTimeout(() => {
+      // Find or create the live region
+      let liveRegion = document.getElementById('claude-announcements');
+      
+      if (!liveRegion) {
+        liveRegion = document.createElement('div');
+        liveRegion.id = 'claude-announcements';
+        liveRegion.setAttribute('aria-live', priority);
+        liveRegion.setAttribute('aria-atomic', 'true');
+        liveRegion.className = 'sr-only';
+        liveRegion.style.cssText = `
+          position: absolute;
+          width: 1px;
+          height: 1px;
+          padding: 0;
+          margin: -1px;
+          overflow: hidden;
+          clip: rect(0, 0, 0, 0);
+          white-space: nowrap;
+          border: 0;
+        `;
+        document.body.appendChild(liveRegion);
+      }
+      
+      // Update the live region priority if needed
+      if (liveRegion.getAttribute('aria-live') !== priority) {
+        liveRegion.setAttribute('aria-live', priority);
+      }
+      
+      // Clear and set the new message
+      liveRegion.textContent = '';
+      // Force a small delay to ensure screen readers detect the change
+      setTimeout(() => {
+        liveRegion!.textContent = message;
+      }, 50);
+    }, delay);
+  }, []);
+
+  /**
+   * Announces when Claude starts responding
+   */
+  const announceClaudeStarted = useCallback(() => {
+    announceMessage('Claude says', { priority: 'polite' });
+  }, [announceMessage]);
+
+  /**
+   * Announces when Claude finishes responding
+   */
+  const announceClaudeFinished = useCallback(() => {
+    announceMessage('Finished', { priority: 'polite' });
+  }, [announceMessage]);
+
+  /**
+   * Announces new assistant message content
+   */
+  const announceAssistantMessage = useCallback((content: string) => {
+    // Clean up the content for screen reader announcement
+    const cleanContent = cleanMessageForAnnouncement(content);
+    if (cleanContent) {
+      announceMessage(`Claude says: ${cleanContent}`, { priority: 'polite' });
+    }
+  }, [announceMessage]);
+
+  /**
+   * Announces tool execution
+   */
+  const announceToolExecution = useCallback((toolName: string, description?: string) => {
+    const message = description 
+      ? `Using ${toolName}: ${description}`
+      : `Using ${toolName}`;
+    announceMessage(message, { priority: 'polite' });
+  }, [announceMessage]);
+
+  /**
+   * Announces tool completion
+   */
+  const announceToolCompleted = useCallback((toolName: string, success: boolean) => {
+    const status = success ? 'done' : 'failed';
+    announceMessage(`${toolName} ${status}`, { priority: 'polite' });
+  }, [announceMessage]);
+
+  return {
+    announceMessage,
+    announceClaudeStarted,
+    announceClaudeFinished,
+    announceAssistantMessage,
+    announceToolExecution,
+    announceToolCompleted,
+  };
+};
+
+/**
+ * Cleans message content for screen reader announcement
+ * Removes markdown formatting and truncates if too long
+ */
+function cleanMessageForAnnouncement(content: string): string {
+  if (!content) return '';
+  
+  // Remove markdown formatting
+  let cleaned = content
+    .replace(/```[\s\S]*?```/g, '[code block]') // Replace code blocks
+    .replace(/`([^`]+)`/g, '$1') // Remove inline code backticks
+    .replace(/\*\*([^*]+)\*\*/g, '$1') // Remove bold formatting
+    .replace(/\*([^*]+)\*/g, '$1') // Remove italic formatting
+    .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') // Replace links with text
+    .replace(/#{1,6}\s*/g, '') // Remove heading markers
+    .replace(/\n+/g, ' ') // Replace newlines with spaces
+    .trim();
+  
+  // Truncate if too long (screen readers work better with shorter announcements)
+  if (cleaned.length > 200) {
+    cleaned = cleaned.substring(0, 197) + '...';
+  }
+  
+  return cleaned;
+}
