Fixed taskbreakdown UI, added generate audio functionality

Author: srujanavanama

azure-functions/jobassistai-http-trigger-openai/system_prompt_mapping.json

@@ -1,5 +1,5 @@
 {
-    "taskbreakdown" : "You are a supportive AI assistant helping job coaches create notes for employee with disabilities in a supported employment program. Be respectful, sensitive and never use stigmatizing language. Break down tasks into simple, numbered steps (Format: 1. [Step] 2. [Step]. Note to Job Coach (Optional). Additional training resources (Optional)) using clear, easy-to-understand language. Steps should be small, specific, and achievable, focusing on one action at a time. Adjust instructions to fit the employee's disability and cognitive abilities.",
+    "taskbreakdown" : "You are a supportive AI assistant helping job coaches create notes for employee with disabilities in a supported employment program. Be respectful, sensitive and never use stigmatizing language. Break down tasks into simple, numbered steps (Format: 1. [Step] 2. [Step]. Note to Job Coach (Optional). Additional training resources (Optional)) using clear, easy-to-understand language. Steps should be small, specific, and achievable, focusing on one action at a time. Adjust instructions to fit the employee's disability and cognitive abilities. Return a structured JSON response where: \"steps_for_employee\" is a single string combining all steps, separated by \" | \", \"note_to_job_coach\" is a single string combining all relevant notes, separated by \" | \", \"additional_training_resources\" is a single string listing resources in \"Resource Name: URL\" format, separated by \" | \". Ensure the response is structured and easy to parse.",
     "search_insights" : "You are a knowledge retention tool for job coaches. Job coaches store random data in Cosmos DB with category and details. This data helps new job coaches answer questions when the original coach is unavailable due to sickness or leave. When a new job coach queries the system, the search results from Cosmos DB are processed using Azure AI Search to retrieve relevant insights. From the retrieved details, extract only the most relevant and concise information that answers the search query. Filter or summarize irrelevant information as needed.",
     "create_insight" : "You are a supportive AI helping job coaches save their knowledge. Given a voice memo transcript, analyze it and return a structured JSON response. Extract the main theme as \"category\" and provide a concise but complete summary as \"details\", ensuring no critical information is lost.",
     "chat": "You are an AI assistant for job placement specialists. You are trained on the same material as the specialists so you understand their roles & responsibilities as well as the mission & values of their organization.Your assistance is invaluable to helping the specialists operate efficiently to improve the lives of their consumers.You may receive general or specific questions and you should helpfully respond accordingly, always using the reference material as your primary source and providing citations as needed.You may also receive structured detail regarding the context in which the question is being asked such as an event that has taken place or detail regarding a consumer.If you receive only the structured detail, respond with the next best action that should be taken based on this detail.If the detail is accompanied by an inquiry, incorporate the detail as context to provide a more targeted response.You should format your response using simple html tags (e.g. ul, li, a, b) that can be embedded in the user's chat window."

demo-app/app.py

@@ -6,6 +6,7 @@
 import sys
 import io
 import time
+import base64
 from azure.storage.blob import BlobServiceClient
 import mimetypes
 import requests
@@ -23,6 +24,7 @@
 FUNCTION_HTTP_OPENAI_WITH_INDEX_URL = os.getenv("FUNCTION_HTTP_OPENAI_WITH_INDEX_URL")
 FUNCTION_SAVE_INSIGHTS_URL = os.getenv("FUNCTION_SAVE_INSIGHTS_URL")
 FUNCTION_SEARCH_INSIGHTS_URL = os.getenv("FUNCTION_SEARCH_INSIGHTS_URL")
+FUNCTION_HTTP_TEXT_TO_SPEECH_URL = os.getenv("FUNCTION_HTTP_TEXT_TO_SPEECH_URL")
 
 SYSTEM_ROLE_TASKBREAKDOWN = os.getenv("SYSTEM_ROLE_TASKBREAKDOWN")
 SYSTEM_ROLE_CHAT = os.getenv("SYSTEM_ROLE_CHAT","chat")
@@ -219,7 +221,7 @@ def download_processed(filename):
         flash(f"Error downloading file: {str(e)}")
         return redirect(url_for('index'))
 
-@app.route('/ai/task-breakdown', methods=['POST'])
+@app.route('/api/task-breakdown', methods=['POST'])
 @login_required
 def ai_task_breakdown():
     try:
@@ -257,27 +259,20 @@ def ai_task_breakdown():
         response = requests.post(FUNCTION_HTTP_OPENAI_URL, json=payload, timeout=30)
 
         if response.status_code == 200:
-            result = response.json()
-            message = result.get('message', '')
+            response_data = response.json()
+            raw_message = response_data.get("message", {})
+            cleaned_response = raw_message.replace("```json", "").replace("```", "").strip()
+            response_message = json.loads(cleaned_response)
 
-            # Parse the response (assuming it’s a string with steps and accommodations)
-            steps = []
-            accommodations_list = []
-            current_list = steps
-            for line in message.split('\n'):
-                line = line.strip()
-                if line.startswith('Steps:') or line.startswith('Step-by-Step Instructions:'):
-                    current_list = steps
-                elif line.startswith('Accommodations:') or line.startswith('Suggested Accommodations:'):
-                    current_list = accommodations_list
-                elif line and line[0].isdigit():
-                    current_list.append(line)
-                elif line.startswith('- ') or line.startswith('* '):
-                    current_list.append(line[2:])
+            # Extract from response_message
+            steps_for_employee = response_message.get("steps_for_employee", "")
+            note_to_job_coach = response_message.get("note_to_job_coach", "")
+            additional_training_resources = response_message.get("additional_training_resources", "")  
 
             return jsonify({
-                'steps': steps,
-                'accommodations': accommodations_list
+                'steps_for_employee': steps_for_employee,
+                'note_to_job_coach': note_to_job_coach,
+                'additional_training_resources': additional_training_resources
             })
         else:
             return jsonify({'error': f"Failed to generate breakdown: {response.text}"}), 500
@@ -286,6 +281,31 @@ def ai_task_breakdown():
         print(f"Error in task breakdown: {str(e)}")
         return jsonify({'error': str(e)}), 500
 
+@app.route('/api/generateAudio', methods=['POST'])
+@login_required
+def generate_audio():
+    print("generate_audio : Generating audio for task breakdown start")
+    try:
+        data = request.json
+        text = data.get('text')
+        if text:
+            payload = {"text" : text}
+            response = requests.post(FUNCTION_HTTP_TEXT_TO_SPEECH_URL, json=payload)
+            if response.status_code == 200:
+                # Encode the audio binary data to base64
+                audio_base64 = base64.b64encode(response.content).decode('utf-8')
+                # Return the audio data in JSON format
+                return jsonify({
+                    'audio': audio_base64
+                })
+            else:
+                return jsonify({'error': 'Failed to generate audio.'}), 400
+        else:
+            return jsonify({'error': 'No text provided for audio generation'}), 400
+    except Exception as e:
+        print(f"Error while generating audio for task breakdown: {str(e)}")
+        return jsonify({'error': str(e)}), 500        
+
 # Add this new route for the AI chat functionality
 @app.route('/api/chat', methods=['POST'])
 @login_required

demo-app/static/css/style.css

@@ -294,7 +294,7 @@ main {
     padding: 1rem;
 }
 
-#taskSteps li, #taskAccommodations li {
+#taskSteps li, #taskNoteToJobCoach li, #taskAdditionalResources li {
     margin-bottom: 0.75rem;
     line-height: 1.5;
 }

demo-app/static/js/script.js

@@ -132,12 +132,28 @@ document.addEventListener('DOMContentLoaded', function() {
     // Task Breakdown Generator
     const generateTaskBtn = document.getElementById('generateTaskBreakdown');
     const taskResultDiv = document.getElementById('taskBreakdownResult');
+    const loadingSpinner = document.getElementById("loadingSpinner");
     const taskStepsList = document.getElementById('taskSteps');
-    const taskAccommodationsList = document.getElementById('taskAccommodations');
+    const taskNoteToJobCoachList = document.getElementById('taskNoteToJobCoach');
+    const taskAdditionalResourcesList = document.getElementById('taskAdditionalResources');
+    const generateAudioButton = document.getElementById('generateAudioButton');
+    const audioLoadingSpinner = document.getElementById('audioLoadingSpinner');
     const saveTaskBtn = document.getElementById('saveTaskBreakdown');
 
     if (generateTaskBtn) {
         generateTaskBtn.addEventListener('click', async function() {
+            // Show loading spinner and hide the result container
+            loadingSpinner.classList.remove('d-none');  // Show the spinner
+            taskResultDiv.classList.add('d-none');
+
+            // Clear the previous audio and hide the audio player
+            const audioPlayer = document.getElementById('audioPlayer');
+            audioPlayer.src = '';  
+            audioPlayer.classList.add('d-none');  
+
+            // Hide the audio loading spinner
+            audioLoadingSpinner.classList.add('d-none');
+
             const taskName = document.getElementById('taskName').value;
             const taskDetails = document.getElementById('taskDetails').value;
             const needsVisual = document.getElementById('needsVisual').checked;
@@ -148,14 +164,8 @@ document.addEventListener('DOMContentLoaded', function() {
                 return;
             }
 
-            // Show loading indicator
-            taskStepsList.innerHTML = '<div class="text-center py-3"><div class="spinner-border text-primary" role="status"></div><p class="mt-2">Generating task breakdown...</p></div>';
-            taskAccommodationsList.innerHTML = '';
-            taskResultDiv.classList.remove('d-none');
-            generateTaskBtn.disabled = true;
-
             try {
-                const response = await fetch('/ai/task-breakdown', {
+                const response = await fetch('/api/task-breakdown', {
                     method: 'POST',
                     headers: { 'Content-Type': 'application/json' },
                     body: JSON.stringify({
@@ -174,45 +184,121 @@ document.addEventListener('DOMContentLoaded', function() {
                 if (response.ok) {
                     // Populate the steps list
                     taskStepsList.innerHTML = '';
-                    result.steps.forEach(step => {
+                    const stepsArray = result.steps_for_employee.split(' | ');
+                    stepsArray.forEach(step => {
                         const li = document.createElement('li');
                         li.className = 'mb-2';
-                        li.textContent = step;
+                        li.textContent = step.replace(/^\d+\.\s*/, ""); ;
                         taskStepsList.appendChild(li);
                     });
 
-                    // Populate the accommodations list
-                    taskAccommodationsList.innerHTML = '';
-                    result.accommodations.forEach(accommodation => {
+                    // Populate the note to job coach list
+                    taskNoteToJobCoachList.innerHTML = '';
+                    const notesArray = result.note_to_job_coach.split(' | ');
+                    notesArray.forEach(note => {
+                        const li = document.createElement('li');
+                        li.className = 'mb-2';
+                        li.textContent = note;
+                        taskNoteToJobCoachList.appendChild(li);
+                    });
+
+                    // Populate the additional resources list
+                    taskAdditionalResourcesList.innerHTML = '';
+                    const resourcesArray = result.additional_training_resources.split(' | '); 
+                    resourcesArray.forEach(resource => {
+                        const [resourceName, resourceLink] = resource.split(': '); 
+
                         const li = document.createElement('li');
                         li.className = 'mb-2';
-                        li.textContent = accommodation;
-                        taskAccommodationsList.appendChild(li);
+
+                        const a = document.createElement('a'); 
+                        a.href = resourceLink.trim(); 
+                        a.textContent = resourceName.trim(); 
+                        a.target = "_blank"; 
+
+                        li.appendChild(a); 
+                        taskAdditionalResourcesList.appendChild(li); 
                     });
 
-                    // Show the save button
+                    // Show the save button, hide the spinner, and display the result
                     saveTaskBtn.classList.remove('d-none');
+                    loadingSpinner.classList.add('d-none');  
+                    taskResultDiv.classList.remove('d-none');
                 } else {
                     taskStepsList.innerHTML = `<p class="text-danger">Error: ${result.error}</p>`;
-                    taskAccommodationsList.innerHTML = '';
+                    loadingSpinner.classList.add('d-none');  
+                    // taskNoteToJobCoachList.innerHTML = '';
+                    // taskAdditionalResourcesList.innerHTML = '';
                 }
             } catch (error) {
                 console.error('Task breakdown error:', error);
                 taskStepsList.innerHTML = '<p class="text-danger">An error occurred while generating the breakdown.</p>';
-                taskAccommodationsList.innerHTML = '';
+                loadingSpinner.classList.add('d-none');  
+                // taskNoteToJobCoachList.innerHTML = '';
+                // taskAdditionalResourcesList.innerHTML = '';
             } finally {
                 generateTaskBtn.disabled = false;
             }
         });
+        generateAudioButton.addEventListener('click', async function() {
+            console.log('Generate audio button clicked');
+            const audioLoadingSpinner = document.getElementById('audioLoadingSpinner');
+            audioLoadingSpinner.classList.remove('d-none');
+            const taskSteps = [];  // Replace with the array containing notes to be converted to audio
+            taskStepsList.querySelectorAll('li').forEach(li => {
+                taskSteps.push(li.textContent);
+            });
+
+            const textToConvert = taskSteps.join(' '); // Combine notes into a single text string
+
+            try {
+                // Call the backend to generate audio
+                const response = await fetch('/api/generateAudio', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({ text: textToConvert })
+                });
+
+                if (response.ok) {
+                    // Parse the JSON response
+                    const data = await response.json();
+
+                    // Get the base64-encoded audio data
+                    const audioBase64 = data.audio;
+
+                    // Convert base64 string to binary data (audio)
+                    const audioBlob = new Blob([new Uint8Array(atob(audioBase64).split("").map(c => c.charCodeAt(0)))], { type: "audio/mpeg" });
+
+                    // Create an audio URL from the blob
+                    const audioUrl = URL.createObjectURL(audioBlob);
+
+                    // Get the audio player element
+                    const audioPlayer = document.getElementById('audioPlayer');
+                    
+                    // Set the audio player source to the blob URL
+                    audioPlayer.src = audioUrl;
 
+                    // Make the audio player visible and remove spinner
+                    audioPlayer.classList.remove('d-none');
+                    audioLoadingSpinner.classList.add('d-none');
+                    // audioPlayer.play();
+                } else {
+                    console.error('Error generating audio:', response.statusText);
+                    audioLoadingSpinner.classList.add('d-none');
+                }
+            } catch (error) {
+                console.error('Error:', error);
+                audioLoadingSpinner.classList.add('d-none');
+            }
+        });
         saveTaskBtn.addEventListener('click', async function() {
             try {
                 const response = await fetch('/upload_note', {
                     method: 'POST',
                     headers: { 'Content-Type': 'application/json' },
                     body: JSON.stringify({
                         category: 'Task Breakdown',
-                        content: `Task: ${document.getElementById('taskName').value}\nSteps:\n${Array.from(taskStepsList.children).map(li => li.textContent).join('\n')}\nAccommodations:\n${Array.from(taskAccommodationsList.children).map(li => li.textContent).join('\n')}`,
+                        content: `Task: ${document.getElementById('taskName').value}\nSteps:\n${Array.from(taskStepsList.children).map(li => li.textContent).join('\n')}\nNote to Job COach:\n${Array.from(taskNoteToJobCoachList.children).map(li => li.textContent).join('\n')}\nAdditional Resources:\n${Array.from(taskAdditionalResourcesList.children).map(li => li.textContent).join('\n')}`,
                         consumer_id: window.consumerId || 'c001'
                     })
                 });
@@ -231,6 +317,33 @@ document.addEventListener('DOMContentLoaded', function() {
         });
     }
 
+    // Listen for when the modal is hidden (closed)
+    $('#taskBreakdownModal').on('hidden.bs.modal', function () {
+        // Reset form fields
+        document.getElementById('taskBreakdownForm').reset();
+        
+        // Reset steps, notes, and resources
+        document.getElementById('taskSteps').innerHTML = '';
+        document.getElementById('taskNoteToJobCoach').innerHTML = '';
+        document.getElementById('taskAdditionalResources').innerHTML = '';
+        
+        // Hide the audio player
+        const audioPlayer = document.getElementById('audioPlayer');
+        audioPlayer.classList.add('d-none');
+        audioPlayer.pause();
+        audioPlayer.src = '';  // Reset the audio source
+        
+        // Hide the loading spinner
+        document.getElementById('loadingSpinner').classList.add('d-none');
+        
+        // Hide the result section
+        document.getElementById('taskBreakdownResult').classList.add('d-none');
+        
+        // Reset any other UI elements
+        document.getElementById('saveTaskBreakdown').classList.add('d-none');
+    });
+
+
     // Add functionality to the AI chat form
     // const aiChatForm = document.getElementById('aiChatForm');
     // if (aiChatForm) {