fix(playground): add better handling and tracking for export to playground errors VSCODE-666 (#906)

Author: gagik

src/participant/participant.ts

@@ -45,7 +45,10 @@ import { processStreamWithIdentifiers } from './streamParsing';
 import type { PromptIntent } from './prompts/intent';
 import { isPlayground, getSelectedText, getAllText } from '../utils/playground';
 import type { DataService } from 'mongodb-data-service';
-import { ParticipantErrorTypes } from './participantErrorTypes';
+import {
+  ParticipantErrorTypes,
+  type ExportToPlaygroundError,
+} from './participantErrorTypes';
 import type PlaygroundResultProvider from '../editors/playgroundResultProvider';
 import { isExportToLanguageResult } from '../types/playgroundType';
 import { PromptHistory } from './prompts/promptHistory';
@@ -345,26 +348,35 @@ export default class ParticipantController {
     token: vscode.CancellationToken;
     language?: string;
   }): Promise<string | null> {
-    const chatResponse = await this._getChatResponse({
-      modelInput,
-      token,
-    });
+    try {
+      const chatResponse = await this._getChatResponse({
+        modelInput,
+        token,
+      });
 
-    const languageCodeBlockIdentifier = {
-      start: `\`\`\`${language ? language : 'javascript'}`,
-      end: '```',
-    };
+      const languageCodeBlockIdentifier = {
+        start: `\`\`\`${language ? language : 'javascript'}`,
+        end: '```',
+      };
 
-    const runnableContent: string[] = [];
-    await processStreamWithIdentifiers({
-      processStreamFragment: () => {},
-      onStreamIdentifier: (content: string) => {
-        runnableContent.push(content.trim());
-      },
-      inputIterable: chatResponse.text,
-      identifier: languageCodeBlockIdentifier,
-    });
-    return runnableContent.length ? runnableContent.join('') : null;
+      const runnableContent: string[] = [];
+      await processStreamWithIdentifiers({
+        processStreamFragment: () => {},
+        onStreamIdentifier: (content: string) => {
+          runnableContent.push(content.trim());
+        },
+        inputIterable: chatResponse.text,
+        identifier: languageCodeBlockIdentifier,
+      });
+      return runnableContent.length ? runnableContent.join('') : null;
+    } catch (error) {
+      /** If anything goes wrong with the response or the stream, return null instead of throwing. */
+      log.error(
+        'Error while streaming chat response with export to language',
+        error
+      );
+      return null;
+    }
   }
 
   async streamChatResponseContentWithCodeActions({
@@ -1784,49 +1796,75 @@ export default class ParticipantController {
   }
 
   async exportCodeToPlayground(): Promise<boolean> {
-    const selectedText = getSelectedText();
-    const codeToExport = selectedText || getAllText();
+    const codeToExport = getSelectedText() || getAllText();
 
     try {
-      const content = await vscode.window.withProgress(
+      const contentOrError = await vscode.window.withProgress<
+        { value: string } | { error: ExportToPlaygroundError }
+      >(
         {
           location: vscode.ProgressLocation.Notification,
           title: 'Exporting code to a playground...',
           cancellable: true,
         },
-        async (progress, token): Promise<string | null> => {
-          const modelInput = await Prompts.exportToPlayground.buildMessages({
-            request: { prompt: codeToExport },
-          });
+        async (
+          progress,
+          token
+        ): Promise<{ value: string } | { error: ExportToPlaygroundError }> => {
+          let modelInput: ModelInput | undefined;
+          try {
+            modelInput = await Prompts.exportToPlayground.buildMessages({
+              request: { prompt: codeToExport },
+            });
+          } catch (error) {
+            return { error: 'modelInput' };
+          }
 
           const result = await Promise.race([
             this.streamChatResponseWithExportToLanguage({
               modelInput,
               token,
             }),
-            new Promise<null>((resolve) =>
+            new Promise<ExportToPlaygroundError>((resolve) =>
               token.onCancellationRequested(() => {
                 log.info('The export to a playground operation was canceled.');
-                resolve(null);
+                resolve('cancelled');
               })
             ),
           ]);
 
-          if (result?.includes("Sorry, I can't assist with that.")) {
-            void vscode.window.showErrorMessage(
-              'Sorry, we were unable to generate the playground, please try again. If the error persists, try changing your selected code.'
-            );
-            return null;
+          if (result === 'cancelled') {
+            return { error: 'cancelled' };
           }
 
-          return result;
+          if (!result || result?.includes("Sorry, I can't assist with that.")) {
+            return { error: 'streamChatResponseWithExportToLanguage' };
+          }
+
+          return { value: result };
         }
       );
 
-      if (!content) {
-        return true;
+      if ('error' in contentOrError) {
+        const { error } = contentOrError;
+        if (error === 'cancelled') {
+          return true;
+        }
+
+        void vscode.window.showErrorMessage(
+          'Failed to generate a MongoDB Playground. Please ensure your code block contains a MongoDB query.'
+        );
+
+        // Content in this case is already equal to the failureType; this is just to make it explicit
+        // and avoid accidentally sending actual contents of the message.
+        this._telemetryService.trackExportToPlaygroundFailed({
+          input_length: codeToExport?.length,
+          error_name: error,
+        });
+        return false;
       }
 
+      const content = contentOrError.value;
       await vscode.commands.executeCommand(
         EXTENSION_COMMANDS.OPEN_PARTICIPANT_CODE_IN_PLAYGROUND,
         {

src/participant/participantErrorTypes.ts

@@ -5,3 +5,8 @@ export enum ParticipantErrorTypes {
   OTHER = 'Other',
   DOCS_CHATBOT_API = 'Docs Chatbot API Issue',
 }
+
+export type ExportToPlaygroundError =
+  | 'cancelled'
+  | 'modelInput'
+  | 'streamChatResponseWithExportToLanguage';

src/telemetry/telemetryService.ts

@@ -12,6 +12,7 @@ import { getConnectionTelemetryProperties } from './connectionTelemetry';
 import type { NewConnectionTelemetryEventProperties } from './connectionTelemetry';
 import type { ShellEvaluateResult } from '../types/playgroundType';
 import type { StorageController } from '../storage';
+import type { ExportToPlaygroundError } from '../participant/participantErrorTypes';
 import { ParticipantErrorTypes } from '../participant/participantErrorTypes';
 import type { ExtensionCommand } from '../commands';
 import type {
@@ -54,6 +55,11 @@ type DocumentEditedTelemetryEventProperties = {
   source: DocumentSource;
 };
 
+type ExportToPlaygroundFailedEventProperties = {
+  input_length: number | undefined;
+  error_name?: ExportToPlaygroundError;
+};
+
 type PlaygroundExportedToLanguageTelemetryEventProperties = {
   language?: string;
   exported_code_length: number;
@@ -106,6 +112,7 @@ type ParticipantResponseFailedProperties = {
   command: ParticipantResponseType;
   error_code?: string;
   error_name: ParticipantErrorTypes;
+  error_details?: string;
 };
 
 export type InternalPromptPurpose = 'intent' | 'namespace' | undefined;
@@ -171,6 +178,7 @@ type TelemetryEventProperties =
   | PlaygroundSavedTelemetryEventProperties
   | PlaygroundLoadedTelemetryEventProperties
   | KeytarSecretsMigrationFailedProperties
+  | ExportToPlaygroundFailedEventProperties
   | SavedConnectionsLoadedProperties
   | ParticipantFeedbackProperties
   | ParticipantResponseFailedProperties
@@ -193,6 +201,7 @@ export enum TelemetryEventTypes {
   PLAYGROUND_EXPORTED_TO_LANGUAGE = 'Playground Exported To Language',
   PLAYGROUND_CREATED = 'Playground Created',
   KEYTAR_SECRETS_MIGRATION_FAILED = 'Keytar Secrets Migration Failed',
+  EXPORT_TO_PLAYGROUND_FAILED = 'Export To Playground Failed',
   SAVED_CONNECTIONS_LOADED = 'Saved Connections Loaded',
   PARTICIPANT_FEEDBACK = 'Participant Feedback',
   PARTICIPANT_WELCOME_SHOWN = 'Participant Welcome Shown',
@@ -346,6 +355,12 @@ export default class TelemetryService {
     );
   }
 
+  trackExportToPlaygroundFailed(
+    props: ExportToPlaygroundFailedEventProperties
+  ): void {
+    this.track(TelemetryEventTypes.EXPORT_TO_PLAYGROUND_FAILED, props);
+  }
+
   trackCommandRun(command: ExtensionCommand): void {
     this.track(TelemetryEventTypes.EXTENSION_COMMAND_RUN, { command });
   }

src/test/suite/participant/participant.test.ts

@@ -1818,6 +1818,37 @@ Schema:
           );
         });
 
+        test('tracks failures with export to playground and not as a failed prompt', async function () {
+          const editor = vscode.window.activeTextEditor;
+          if (!editor) {
+            throw new Error('Window active text editor is undefined');
+          }
+
+          const testDocumentUri = editor.document.uri;
+          const edit = new vscode.WorkspaceEdit();
+          const code = `
+          THIS IS SOME ERROR CAUSING CODE.
+`;
+          edit.replace(testDocumentUri, getFullRange(editor.document), code);
+          await vscode.workspace.applyEdit(edit);
+
+          await testParticipantController.exportCodeToPlayground();
+          sendRequestStub.rejects();
+          const messages = sendRequestStub.firstCall.args[0];
+          expect(getMessageContent(messages[1])).to.equal(code.trim());
+          expect(telemetryTrackStub).calledWith(
+            TelemetryEventTypes.EXPORT_TO_PLAYGROUND_FAILED,
+            {
+              input_length: code.trim().length,
+              error_name: 'streamChatResponseWithExportToLanguage',
+            }
+          );
+
+          expect(telemetryTrackStub).not.calledWith(
+            TelemetryEventTypes.PARTICIPANT_RESPONSE_FAILED
+          );
+        });
+
         test('exports selected lines of code to a playground', async function () {
           const editor = vscode.window.activeTextEditor;
           if (!editor) {