From 6bdd4bd0d3fccc2f7776a33be6edff4c026b96d8 Mon Sep 17 00:00:00 2001 From: Nikola Irinchev Date: Wed, 22 Jan 2025 17:55:01 +0100 Subject: [PATCH 1/3] chore(telemetry): refactor telemetry types --- src/connectionController.ts | 36 +- src/documentSource.ts | 2 + src/editors/editorsController.ts | 2 +- src/editors/mongoDBDocumentService.ts | 15 +- src/editors/playgroundController.ts | 68 +- .../queryWithCopilotCodeLensProvider.ts | 1 + src/explorer/helpExplorer.ts | 2 +- src/explorer/helpTree.ts | 7 +- src/mdbExtensionController.ts | 14 +- src/participant/participant.ts | 182 +++-- src/participant/participantTypes.ts | 9 +- src/participant/prompts/docs.ts | 2 +- src/participant/prompts/intent.ts | 2 +- src/participant/prompts/namespace.ts | 2 +- src/participant/prompts/promptBase.ts | 12 +- src/telemetry/connectionTelemetry.ts | 6 +- src/telemetry/index.ts | 3 +- src/telemetry/telemetryEvents.ts | 666 ++++++++++++++++++ src/telemetry/telemetryService.ts | 388 +--------- src/test/suite/connectionController.test.ts | 9 +- .../activeConnectionCodeLensProvider.test.ts | 2 +- .../collectionDocumentsProvider.test.ts | 6 +- .../editDocumentCodeLensProvider.test.ts | 2 +- .../exportToLanguageCodeLensProvider.test.ts | 2 +- .../editors/mongoDBDocumentService.test.ts | 2 +- .../editors/playgroundController.test.ts | 2 +- .../editors/playgroundResultProvider.test.ts | 2 +- src/test/suite/explorer/helpExplorer.test.ts | 17 +- .../language/languageServerController.test.ts | 2 +- src/test/suite/oidc.test.ts | 2 +- .../suite/participant/participant.test.ts | 207 +++--- .../suite/telemetry/telemetryService.test.ts | 118 ++-- .../suite/views/webviewController.test.ts | 2 +- src/utils/playground.ts | 14 - src/views/webviewController.ts | 16 +- 35 files changed, 1102 insertions(+), 722 deletions(-) create mode 100644 src/telemetry/telemetryEvents.ts diff --git a/src/connectionController.ts b/src/connectionController.ts index 159ecea8e..cbb3c8149 100644 --- a/src/connectionController.ts +++ b/src/connectionController.ts @@ -19,7 +19,7 @@ import { createLogger } from './logging'; import formatError from './utils/formatError'; import type { StorageController } from './storage'; import type { StatusView } from './views'; -import type TelemetryService from './telemetry/telemetryService'; +import type TelemetryService from './telemetry'; import { openLink } from './utils/linkHelper'; import type { ConnectionSource, @@ -30,6 +30,7 @@ import LINKS from './utils/links'; import { isAtlasStream } from 'mongodb-build-info'; import { DocumentSource } from './documentSource'; import type { ConnectionTreeItem } from './explorer'; +import { PresetConnectionEditedTelemetryEvent } from './telemetry'; // eslint-disable-next-line @typescript-eslint/no-var-requires const packageJSON = require('../package.json'); @@ -129,10 +130,12 @@ export default class ConnectionController { // have a setting on the system for storing credentials. // When the setting is on this `connectionMergeInfos` would have the session // credential information and merge it before connecting. - connectionMergeInfos: Record> = - Object.create(null); + private _connectionMergeInfos: Record< + string, + RecursivePartial + > = Object.create(null); - _activeDataService: DataService | null = null; + private _activeDataService: DataService | null = null; _connectionStorage: ConnectionStorage; _telemetryService: TelemetryService; @@ -140,7 +143,7 @@ export default class ConnectionController { private _currentConnectionId: null | string = null; _connectionAttempt: null | ConnectionAttempt = null; - _connectionStringInputCancellationToken: null | vscode.CancellationTokenSource = + private _connectionStringInputCancellationToken: null | vscode.CancellationTokenSource = null; private _connectingConnectionId: null | string = null; private _disconnecting = false; @@ -169,10 +172,11 @@ export default class ConnectionController { async openPresetConnectionsSettings( originTreeItem: ConnectionTreeItem | undefined ): Promise { - this._telemetryService.trackPresetConnectionEdited({ - source: DocumentSource.DOCUMENT_SOURCE_TREEVIEW, - source_details: originTreeItem ? 'tree_item' : 'header', - }); + this._telemetryService.track( + new PresetConnectionEditedTelemetryEvent( + originTreeItem ? 'tree_item' : 'header' + ) + ); let source: ConnectionSource | undefined = originTreeItem?.source; if (!source) { const mdbConfiguration = vscode.workspace.getConfiguration('mdb'); @@ -227,7 +231,7 @@ export default class ConnectionController { // TODO: re-enable with fewer 'Saved Connections Loaded' events // https://jira.mongodb.org/browse/VSCODE-462 - /* this._telemetryService.trackSavedConnectionsLoaded({ + /* this._telemetryService.track(new SavedConnectionsLoadedTelemetryEvent({ saved_connections: globalAndWorkspaceConnections.length, loaded_connections: loadedConnections.length, ).length, @@ -236,7 +240,7 @@ export default class ConnectionController { connection.secretStorageLocation === SecretStorageLocation.SecretStorage ).length, - }); */ + })); */ } async connectWithURI(): Promise { @@ -402,7 +406,7 @@ export default class ConnectionController { const connectionInfo: LoadedConnection = merge( cloneDeep(this._connections[connectionId]), - this.connectionMergeInfos[connectionId] ?? {} + this._connectionMergeInfos[connectionId] ?? {} ); if (!connectionInfo.connectionOptions) { @@ -557,8 +561,8 @@ export default class ConnectionController { mergeConnectionInfo = { connectionOptions: await dataService.getUpdatedSecrets(), }; - this.connectionMergeInfos[connectionInfo.id] = merge( - cloneDeep(this.connectionMergeInfos[connectionInfo.id]), + this._connectionMergeInfos[connectionInfo.id] = merge( + cloneDeep(this._connectionMergeInfos[connectionInfo.id]), mergeConnectionInfo ); } @@ -585,8 +589,8 @@ export default class ConnectionController { connectionOptions: await dataService.getUpdatedSecrets(), }; if (!mergeConnectionInfo) return; - this.connectionMergeInfos[connectionInfo.id] = merge( - cloneDeep(this.connectionMergeInfos[connectionInfo.id]), + this._connectionMergeInfos[connectionInfo.id] = merge( + cloneDeep(this._connectionMergeInfos[connectionInfo.id]), mergeConnectionInfo ); diff --git a/src/documentSource.ts b/src/documentSource.ts index 329032a86..066c96cb6 100644 --- a/src/documentSource.ts +++ b/src/documentSource.ts @@ -4,3 +4,5 @@ export enum DocumentSource { DOCUMENT_SOURCE_COLLECTIONVIEW = 'collectionview', DOCUMENT_SOURCE_CODELENS = 'codelens', } + +export type DocumentSourceDetails = 'database' | 'collection' | undefined; diff --git a/src/editors/editorsController.ts b/src/editors/editorsController.ts index 66f1024c5..74d8f8008 100644 --- a/src/editors/editorsController.ts +++ b/src/editors/editorsController.ts @@ -31,7 +31,7 @@ import type PlaygroundController from './playgroundController'; import type PlaygroundResultProvider from './playgroundResultProvider'; import { PLAYGROUND_RESULT_SCHEME } from './playgroundResultProvider'; import { StatusView } from '../views'; -import type TelemetryService from '../telemetry/telemetryService'; +import type TelemetryService from '../telemetry'; import type { QueryWithCopilotCodeLensProvider } from './queryWithCopilotCodeLensProvider'; const log = createLogger('editors controller'); diff --git a/src/editors/mongoDBDocumentService.ts b/src/editors/mongoDBDocumentService.ts index 1806cddda..e1d81d2de 100644 --- a/src/editors/mongoDBDocumentService.ts +++ b/src/editors/mongoDBDocumentService.ts @@ -7,8 +7,9 @@ import { DocumentSource } from '../documentSource'; import type { EditDocumentInfo } from '../types/editDocumentInfoType'; import formatError from '../utils/formatError'; import type { StatusView } from '../views'; -import type TelemetryService from '../telemetry/telemetryService'; +import type TelemetryService from '../telemetry'; import { getEJSON } from '../utils/ejson'; +import { DocumentUpdatedTelemetryEvent } from '../telemetry'; const log = createLogger('document controller'); @@ -50,9 +51,11 @@ export default class MongoDBDocumentService { _saveDocumentFailed(message: string): void { const errorMessage = `Unable to save document: ${message}`; - this._telemetryService.trackDocumentUpdated( - DocumentSource.DOCUMENT_SOURCE_TREEVIEW, - false + this._telemetryService.track( + new DocumentUpdatedTelemetryEvent( + DocumentSource.DOCUMENT_SOURCE_TREEVIEW, + false + ) ); throw new Error(errorMessage); @@ -98,7 +101,9 @@ export default class MongoDBDocumentService { returnDocument: 'after', } ); - this._telemetryService.trackDocumentUpdated(source, true); + this._telemetryService.track( + new DocumentUpdatedTelemetryEvent(source, true) + ); } catch (error) { return this._saveDocumentFailed(formatError(error).message); } finally { diff --git a/src/editors/playgroundController.ts b/src/editors/playgroundController.ts index 7f2c2969d..f89dbbe53 100644 --- a/src/editors/playgroundController.ts +++ b/src/editors/playgroundController.ts @@ -34,16 +34,16 @@ import { import playgroundSearchTemplate from '../templates/playgroundSearchTemplate'; import playgroundTemplate from '../templates/playgroundTemplate'; import type { StatusView } from '../views'; -import type TelemetryService from '../telemetry/telemetryService'; -import { - isPlayground, - getSelectedText, - getAllText, - getPlaygroundExtensionForTelemetry, -} from '../utils/playground'; +import type TelemetryService from '../telemetry'; +import { isPlayground, getSelectedText, getAllText } from '../utils/playground'; import type ExportToLanguageCodeLensProvider from './exportToLanguageCodeLensProvider'; import { playgroundFromDatabaseTreeItemTemplate } from '../templates/playgroundFromDatabaseTreeItemTemplate'; import { playgroundFromCollectionTreeItemTemplate } from '../templates/playgroundFromCollectionTreeItemTemplate'; +import { + PlaygroundCreatedTelemetryEvent, + PlaygroundExecutedTelemetryEvent, + PlaygroundSavedTelemetryEvent, +} from '../telemetry'; const log = createLogger('playground controller'); @@ -135,17 +135,15 @@ export default class PlaygroundController { if (isPlayground(document.uri)) { // TODO: re-enable with fewer 'Playground Loaded' events // https://jira.mongodb.org/browse/VSCODE-432 - /* this._telemetryService.trackPlaygroundLoaded( - getPlaygroundExtensionForTelemetry(document.uri) - ); */ + // this._telemetryService.track(new PlaygroundLoadedTelemetryEvent(document.uri)); await vscode.languages.setTextDocumentLanguage(document, 'javascript'); } }); vscode.workspace.onDidSaveTextDocument((document) => { if (isPlayground(document.uri)) { - this._telemetryService.trackPlaygroundSaved( - getPlaygroundExtensionForTelemetry(document.uri) + this._telemetryService.track( + new PlaygroundSavedTelemetryEvent(document.uri) ); } }); @@ -231,7 +229,7 @@ export default class PlaygroundController { .replace('CURRENT_DATABASE', databaseName) .replace('CURRENT_COLLECTION', collectionName); - this._telemetryService.trackPlaygroundCreated('search'); + this._telemetryService.track(new PlaygroundCreatedTelemetryEvent('search')); return this._createPlaygroundFileWithContent(content); } @@ -246,9 +244,13 @@ export default class PlaygroundController { content = content .replace('NEW_DATABASE_NAME', element.databaseName) .replace('Create a new database', 'The current database to use'); - this._telemetryService.trackPlaygroundCreated('createCollection'); + this._telemetryService.track( + new PlaygroundCreatedTelemetryEvent('createCollection') + ); } else { - this._telemetryService.trackPlaygroundCreated('createDatabase'); + this._telemetryService.track( + new PlaygroundCreatedTelemetryEvent('createDatabase') + ); } return this._createPlaygroundFileWithContent(content); @@ -262,7 +264,7 @@ export default class PlaygroundController { .replace('CURRENT_DATABASE', databaseName) .replace('CURRENT_COLLECTION', collectionName); - this._telemetryService.trackPlaygroundCreated('index'); + this._telemetryService.track(new PlaygroundCreatedTelemetryEvent('index')); return this._createPlaygroundFileWithContent(content); } @@ -277,7 +279,7 @@ export default class PlaygroundController { const content = useDefaultTemplate ? playgroundBasicTextTemplate.replace('PLAYGROUND_CONTENT', text) : text; - this._telemetryService.trackPlaygroundCreated('agent'); + this._telemetryService.track(new PlaygroundCreatedTelemetryEvent('agent')); return this._createPlaygroundFileWithContent(content); } @@ -291,7 +293,9 @@ export default class PlaygroundController { .replace('CURRENT_COLLECTION', collectionName) .replace('DOCUMENT_CONTENTS', documentContents); - this._telemetryService.trackPlaygroundCreated('cloneDocument'); + this._telemetryService.track( + new PlaygroundCreatedTelemetryEvent('cloneDocument') + ); return this._createPlaygroundFileWithContent(content); } @@ -303,7 +307,9 @@ export default class PlaygroundController { .replace('CURRENT_DATABASE', databaseName) .replace('CURRENT_COLLECTION', collectionName); - this._telemetryService.trackPlaygroundCreated('insertDocument'); + this._telemetryService.track( + new PlaygroundCreatedTelemetryEvent('insertDocument') + ); return this._createPlaygroundFileWithContent(content); } @@ -314,7 +320,9 @@ export default class PlaygroundController { element.cacheIsUpToDate = false; - this._telemetryService.trackPlaygroundCreated('createStreamProcessor'); + this._telemetryService.track( + new PlaygroundCreatedTelemetryEvent('createStreamProcessor') + ); return this._createPlaygroundFileWithContent(content); } @@ -325,13 +333,17 @@ export default class PlaygroundController { let content = ''; if (treeItem instanceof DatabaseTreeItem) { content = playgroundFromDatabaseTreeItemTemplate(treeItem.databaseName); - this._telemetryService.trackPlaygroundCreated('fromDatabaseTreeItem'); + this._telemetryService.track( + new PlaygroundCreatedTelemetryEvent('fromDatabaseTreeItem') + ); } else if (treeItem instanceof CollectionTreeItem) { content = playgroundFromCollectionTreeItemTemplate( treeItem.databaseName, treeItem.collectionName ); - this._telemetryService.trackPlaygroundCreated('fromCollectionTreeItem'); + this._telemetryService.track( + new PlaygroundCreatedTelemetryEvent('fromCollectionTreeItem') + ); } return this._createPlaygroundFileWithContent(content); @@ -350,7 +362,7 @@ export default class PlaygroundController { content = template; } - this._telemetryService.trackPlaygroundCreated('crud'); + this._telemetryService.track(new PlaygroundCreatedTelemetryEvent('crud')); return this._createPlaygroundFileWithContent(content); } @@ -391,10 +403,12 @@ export default class PlaygroundController { } this._statusView.hideMessage(); - this._telemetryService.trackPlaygroundCodeExecuted( - result, - this._isPartialRun, - result ? false : true + this._telemetryService.track( + new PlaygroundExecutedTelemetryEvent( + result, + this._isPartialRun, + result ? false : true + ) ); return result; diff --git a/src/editors/queryWithCopilotCodeLensProvider.ts b/src/editors/queryWithCopilotCodeLensProvider.ts index 993d718a7..d1a95c7f7 100644 --- a/src/editors/queryWithCopilotCodeLensProvider.ts +++ b/src/editors/queryWithCopilotCodeLensProvider.ts @@ -35,6 +35,7 @@ export class QueryWithCopilotCodeLensProvider isNewChat: true, telemetry: { source: DocumentSource.DOCUMENT_SOURCE_CODELENS, + source_details: undefined, }, }; diff --git a/src/explorer/helpExplorer.ts b/src/explorer/helpExplorer.ts index 2e6cdf2fb..1dbf3b374 100644 --- a/src/explorer/helpExplorer.ts +++ b/src/explorer/helpExplorer.ts @@ -1,6 +1,6 @@ import * as vscode from 'vscode'; import HelpTree from './helpTree'; -import type { TelemetryService } from '../telemetry'; +import type TelemetryService from '../telemetry'; export default class HelpExplorer { _treeController: HelpTree; diff --git a/src/explorer/helpTree.ts b/src/explorer/helpTree.ts index 5d54c97e8..dc64d981f 100644 --- a/src/explorer/helpTree.ts +++ b/src/explorer/helpTree.ts @@ -1,9 +1,10 @@ import * as vscode from 'vscode'; import path from 'path'; import { getImagesPath } from '../extensionConstants'; -import type { TelemetryService } from '../telemetry'; +import type TelemetryService from '../telemetry'; import { openLink } from '../utils/linkHelper'; import LINKS from '../utils/links'; +import { LinkClickedTelemetryEvent } from '../telemetry'; const HELP_LINK_CONTEXT_VALUE = 'HELP_LINK'; @@ -144,7 +145,9 @@ export default class HelpTree telemetryService: TelemetryService ): Promise { if (helpItem.contextValue === HELP_LINK_CONTEXT_VALUE) { - telemetryService.trackLinkClicked('helpPanel', helpItem.linkId); + telemetryService.track( + new LinkClickedTelemetryEvent('helpPanel', helpItem.linkId) + ); if (helpItem.useRedirect) { try { diff --git a/src/mdbExtensionController.ts b/src/mdbExtensionController.ts index b6f0b9505..1432d01f7 100644 --- a/src/mdbExtensionController.ts +++ b/src/mdbExtensionController.ts @@ -33,7 +33,7 @@ import launchMongoShell from './commands/launchMongoShell'; import type SchemaTreeItem from './explorer/schemaTreeItem'; import { StatusView } from './views'; import { StorageController, StorageVariables } from './storage'; -import TelemetryService from './telemetry/telemetryService'; +import TelemetryService from './telemetry'; import type PlaygroundsTreeItem from './explorer/playgroundsTreeItem'; import PlaygroundResultProvider from './editors/playgroundResultProvider'; import WebviewController from './views/webviewController'; @@ -51,6 +51,10 @@ import type { } from './participant/participantTypes'; import EXTENSION_COMMANDS from './commands'; import { COPILOT_EXTENSION_ID } from './participant/constants'; +import { + CommandRunTelemetryEvent, + DocumentEditedTelemetryEvent, +} from './telemetry'; // This class is the top-level controller for our extension. // Commands which the extensions handles are defined in the function `activate`. @@ -313,7 +317,9 @@ export default class MDBExtensionController implements vscode.Disposable { this.registerCommand( EXTENSION_COMMANDS.MDB_OPEN_MONGODB_DOCUMENT_FROM_CODE_LENS, (data: EditDocumentInfo) => { - this._telemetryService.trackDocumentOpenedInEditor(data.source); + this._telemetryService.track( + new DocumentEditedTelemetryEvent(data.source) + ); return this._editorsController.openMongoDBDocument(data); } @@ -408,7 +414,7 @@ export default class MDBExtensionController implements vscode.Disposable { commandHandler: (...args: any[]) => Promise ): void => { const commandHandlerWithTelemetry = (args: any[]): Promise => { - this._telemetryService.trackCommandRun(command); + this._telemetryService.track(new CommandRunTelemetryEvent(command)); return commandHandler(args); }; @@ -426,7 +432,7 @@ export default class MDBExtensionController implements vscode.Disposable { commandHandler: (...args: any[]) => Promise ): void => { const commandHandlerWithTelemetry = (args: any[]): Promise => { - this._telemetryService.trackCommandRun(command); + this._telemetryService.track(new CommandRunTelemetryEvent(command)); return commandHandler(args); }; diff --git a/src/participant/participant.ts b/src/participant/participant.ts index f54f5ff83..9b57cfaa8 100644 --- a/src/participant/participant.ts +++ b/src/participant/participant.ts @@ -34,11 +34,19 @@ import { type OpenSchemaCommandArgs, } from './prompts/schema'; import { - chatResultFeedbackKindToTelemetryValue, - TelemetryEventTypes, -} from '../telemetry/telemetryService'; + ExportToPlaygroundFailedTelemetryEvent, + ParticipantChatOpenedFromActionTelemetryEvent, + ParticipantFeedbackTelemetryEvent, + ParticipantInputBoxSubmittedTelemetryEvent, + ParticipantPromptSubmittedFromActionTelemetryEvent, + ParticipantPromptSubmittedTelemetryEvent, + ParticipantResponseFailedTelemetryEvent, + ParticipantResponseGeneratedTelemetryEvent, + ParticipantWelcomeShownTelemetryEvent, + PlaygroundExportedToLanguageTelemetryEvent, +} from '../telemetry'; import { DocsChatbotAIService } from './docsChatbotAIService'; -import type TelemetryService from '../telemetry/telemetryService'; +import type TelemetryService from '../telemetry'; import formatError from '../utils/formatError'; import { getContent, type ModelInput } from './prompts/promptBase'; import { processStreamWithIdentifiers } from './streamParsing'; @@ -163,17 +171,18 @@ export default class ParticipantController { if (telemetry) { if (isNewChat) { - this._telemetryService.trackParticipantChatOpenedFromAction({ - ...telemetry, - command, - }); + this._telemetryService.track( + new ParticipantChatOpenedFromActionTelemetryEvent(telemetry, command) + ); } if (!isPartialQuery) { - this._telemetryService.trackParticipantPromptSubmittedFromAction({ - ...telemetry, - command: command ?? 'generic', - input_length: query.length, - }); + this._telemetryService.track( + new ParticipantPromptSubmittedFromActionTelemetryEvent( + telemetry, + command ?? 'generic', + query.length + ) + ); } } @@ -200,12 +209,13 @@ export default class ParticipantController { }); if (telemetry) { - this._telemetryService.trackParticipantInputBoxSubmitted({ - ...telemetry, - input_length: message?.length, - dismissed: message === undefined, - command, - }); + this._telemetryService.track( + new ParticipantInputBoxSubmittedTelemetryEvent( + telemetry, + message, + command + ) + ); } if (message === undefined || message.trim() === '') { @@ -273,7 +283,9 @@ export default class ParticipantController { }) ), }); - this._telemetryService.trackParticipantPrompt(modelInput.stats); + this._telemetryService.track( + new ParticipantPromptSubmittedTelemetryEvent(modelInput.stats) + ); const modelResponse = await model.sendRequest( modelInput.messages, @@ -462,13 +474,15 @@ export default class ParticipantController { stream, }); - this._telemetryService.trackParticipantResponse({ - command: 'generic', - has_cta: false, - found_namespace: false, - has_runnable_content: hasCodeBlock, - output_length: outputLength, - }); + this._telemetryService.track( + new ParticipantResponseGeneratedTelemetryEvent({ + command: 'generic', + hasCta: false, + foundNamespace: false, + hasRunnableContent: hasCodeBlock, + outputLength: outputLength, + }) + ); return genericRequestChatResult(context.history); } @@ -1429,13 +1443,15 @@ export default class ParticipantController { ], }); - this._telemetryService.trackParticipantResponse({ - command: 'schema', - has_cta: true, - found_namespace: true, - has_runnable_content: false, - output_length: response.outputLength, - }); + this._telemetryService.track( + new ParticipantResponseGeneratedTelemetryEvent({ + command: 'schema', + hasCta: true, + foundNamespace: true, + hasRunnableContent: false, + outputLength: response.outputLength, + }) + ); return schemaRequestChatResult(context.history); } @@ -1540,13 +1556,15 @@ export default class ParticipantController { token, }); - this._telemetryService.trackParticipantResponse({ - command: 'query', - has_cta: false, - found_namespace: true, - has_runnable_content: hasCodeBlock, - output_length: outputLength, - }); + this._telemetryService.track( + new ParticipantResponseGeneratedTelemetryEvent({ + command: 'query', + hasCta: false, + foundNamespace: true, + hasRunnableContent: hasCodeBlock, + outputLength: outputLength, + }) + ); return queryRequestChatResult(context.history); } @@ -1612,7 +1630,9 @@ export default class ParticipantController { const stats = Prompts.docs.getStats(history, { request, context }); - this._telemetryService.trackParticipantPrompt(stats); + this._telemetryService.track( + new ParticipantPromptSubmittedTelemetryEvent(stats) + ); log.info('Docs chatbot message sent', { chatId, @@ -1651,13 +1671,15 @@ export default class ParticipantController { this._streamGenericDocsLink(stream); - this._telemetryService.trackParticipantResponse({ - command: 'docs/copilot', - has_cta: true, - found_namespace: false, - has_runnable_content: hasCodeBlock, - output_length: outputLength, - }); + this._telemetryService.track( + new ParticipantResponseGeneratedTelemetryEvent({ + command: 'docs/copilot', + hasCta: true, + foundNamespace: false, + hasRunnableContent: hasCodeBlock, + outputLength: outputLength, + }) + ); } _streamResponseReference({ @@ -1731,13 +1753,15 @@ export default class ParticipantController { } } - this._telemetryService.trackParticipantResponse({ - command: 'docs/chatbot', - has_cta: !!docsResult.responseReferences, - found_namespace: false, - has_runnable_content: false, - output_length: docsResult.responseContent?.length ?? 0, - }); + this._telemetryService.track( + new ParticipantResponseGeneratedTelemetryEvent({ + command: 'docs/chatbot', + hasCta: !!docsResult.responseReferences, + foundNamespace: false, + hasRunnableContent: false, + outputLength: docsResult.responseContent?.length ?? 0, + }) + ); } catch (error) { // If the docs chatbot API is not available, fall back to Copilot’s LLM and include // the MongoDB documentation link for users to go to our documentation site directly. @@ -1751,11 +1775,10 @@ export default class ParticipantController { } this._telemetryService.track( - TelemetryEventTypes.PARTICIPANT_RESPONSE_FAILED, - { - command: 'docs', - error_name: ParticipantErrorTypes.DOCS_CHATBOT_API, - } + new ParticipantResponseFailedTelemetryEvent( + 'docs', + ParticipantErrorTypes.DOCS_CHATBOT_API + ) ); await this._handleDocsRequestWithCopilot(...args); @@ -1857,10 +1880,12 @@ export default class ParticipantController { // Content in this case is already equal to the failureType; this is just to make it explicit // and avoid accidentally sending actual contents of the message. - this._telemetryService.trackExportToPlaygroundFailed({ - input_length: codeToExport?.length, - error_name: error, - }); + this._telemetryService.track( + new ExportToPlaygroundFailedTelemetryEvent( + codeToExport?.length, + error + ) + ); return false; } @@ -1957,7 +1982,7 @@ Please see our [FAQ](https://www.mongodb.com/docs/generative-ai-faq/) for more i ); this._telemetryService.track( - TelemetryEventTypes.PARTICIPANT_WELCOME_SHOWN + new ParticipantWelcomeShownTelemetryEvent() ); await this._storageController.update( @@ -2033,11 +2058,13 @@ Please see our [FAQ](https://www.mongodb.com/docs/generative-ai-faq/) for more i 'unhelpfulReason' in feedback ? (feedback.unhelpfulReason as string) : undefined; - this._telemetryService.trackParticipantFeedback({ - feedback: chatResultFeedbackKindToTelemetryValue(feedback.kind), - reason: unhelpfulReason, - response_type: (feedback.result as ChatResult)?.metadata.intent, - }); + this._telemetryService.track( + new ParticipantFeedbackTelemetryEvent( + feedback.kind, + (feedback.result as ChatResult)?.metadata.intent, + unhelpfulReason + ) + ); } _getConnectionNames(): string[] { @@ -2119,11 +2146,14 @@ Please see our [FAQ](https://www.mongodb.com/docs/generative-ai-faq/) for more i language, includeDriverSyntax, }); - this._telemetryService.trackPlaygroundExportedToLanguageExported({ - language, - exported_code_length: transpiledContent?.length || 0, - with_driver_syntax: includeDriverSyntax, - }); + + this._telemetryService.track( + new PlaygroundExportedToLanguageTelemetryEvent( + language, + transpiledContent?.length, + includeDriverSyntax + ) + ); await vscode.commands.executeCommand( EXTENSION_COMMANDS.SHOW_EXPORT_TO_LANGUAGE_RESULT, diff --git a/src/participant/participantTypes.ts b/src/participant/participantTypes.ts index b8fdc257f..3b65ceef0 100644 --- a/src/participant/participantTypes.ts +++ b/src/participant/participantTypes.ts @@ -1,5 +1,5 @@ import type * as vscode from 'vscode'; -import type { DocumentSource } from '../documentSource'; +import type { ParticipantTelemetryMetadata } from '../telemetry'; export type ParticipantCommandType = 'query' | 'schema' | 'docs'; export type ParticipantCommand = `/${ParticipantCommandType}`; @@ -19,18 +19,13 @@ export type ParticipantResponseType = | 'askToConnect' | 'askForNamespace'; -type TelemetryMetadata = { - source: DocumentSource; - source_details?: 'database' | 'collection'; -}; - /** Based on options from Copilot's chat open command IChatViewOpenOptions */ export type SendMessageToParticipantOptions = { message: string; command?: ParticipantCommandType; isNewChat?: boolean; isPartialQuery?: boolean; - telemetry?: TelemetryMetadata; + telemetry?: ParticipantTelemetryMetadata; }; export type SendMessageToParticipantFromInputOptions = Pick< diff --git a/src/participant/prompts/docs.ts b/src/participant/prompts/docs.ts index 316e12809..41b1cfe59 100644 --- a/src/participant/prompts/docs.ts +++ b/src/participant/prompts/docs.ts @@ -1,4 +1,4 @@ -import type { ParticipantPromptProperties } from '../../telemetry/telemetryService'; +import type { ParticipantPromptProperties } from '../../telemetry'; import type { PromptArgsBase } from './promptBase'; import { PromptBase } from './promptBase'; import type * as vscode from 'vscode'; diff --git a/src/participant/prompts/intent.ts b/src/participant/prompts/intent.ts index 8a1266f69..02096fbd4 100644 --- a/src/participant/prompts/intent.ts +++ b/src/participant/prompts/intent.ts @@ -1,4 +1,4 @@ -import type { InternalPromptPurpose } from '../../telemetry/telemetryService'; +import type { InternalPromptPurpose } from '../../telemetry'; import type { PromptArgsBase } from './promptBase'; import { PromptBase } from './promptBase'; diff --git a/src/participant/prompts/namespace.ts b/src/participant/prompts/namespace.ts index c5428f191..2cbd5bdbb 100644 --- a/src/participant/prompts/namespace.ts +++ b/src/participant/prompts/namespace.ts @@ -1,4 +1,4 @@ -import type { InternalPromptPurpose } from '../../telemetry/telemetryService'; +import type { InternalPromptPurpose } from '../../telemetry'; import type { PromptArgsBase } from './promptBase'; import { PromptBase } from './promptBase'; diff --git a/src/participant/prompts/promptBase.ts b/src/participant/prompts/promptBase.ts index 4bde40d67..e49bd6775 100644 --- a/src/participant/prompts/promptBase.ts +++ b/src/participant/prompts/promptBase.ts @@ -3,7 +3,7 @@ import type { ChatResult } from '../constants'; import type { InternalPromptPurpose, ParticipantPromptProperties, -} from '../../telemetry/telemetryService'; +} from '../../telemetry'; import { PromptHistory } from './promptHistory'; import type { ParticipantCommandType } from '../participantTypes'; import { getCopilotModel } from '../model'; @@ -208,15 +208,15 @@ export abstract class PromptBase { hasSampleDocs: boolean ): ParticipantPromptProperties { return { - total_message_length: messages.reduce( + totalMessageLength: messages.reduce( (acc, message) => acc + getContentLength(message), 0 ), - user_input_length: request.prompt.length, - has_sample_documents: hasSampleDocs, + userInputLength: request.prompt.length, + hasSampleDocuments: hasSampleDocs, command: (request.command as ParticipantCommandType) || 'generic', - history_size: context?.history.length || 0, - internal_purpose: this.internalPurposeForTelemetry, + historySize: context?.history.length || 0, + internalPurpose: this.internalPurposeForTelemetry, }; } } diff --git a/src/telemetry/connectionTelemetry.ts b/src/telemetry/connectionTelemetry.ts index c92f675ac..ecdbf37ed 100644 --- a/src/telemetry/connectionTelemetry.ts +++ b/src/telemetry/connectionTelemetry.ts @@ -39,13 +39,13 @@ export type HostInformation = { function getHostnameForConnection(dataService: DataService): string | null { const lastSeenTopology = dataService.getLastSeenTopology(); - const resolvedHost = lastSeenTopology?.servers.values().next().value.address; + const resolvedHost = lastSeenTopology?.servers.values().next().value?.address; - if (resolvedHost.startsWith('[')) { + if (resolvedHost?.startsWith('[')) { return resolvedHost.slice(1).split(']')[0]; // IPv6 } - return resolvedHost.split(':')[0]; + return resolvedHost?.split(':')[0] || null; } async function getPublicCloudInfo(host: string): Promise<{ diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts index 7fc815c66..a9495a57d 100644 --- a/src/telemetry/index.ts +++ b/src/telemetry/index.ts @@ -1,3 +1,4 @@ import TelemetryService from './telemetryService'; +export * from './telemetryEvents'; -export { TelemetryService }; +export default TelemetryService; diff --git a/src/telemetry/telemetryEvents.ts b/src/telemetry/telemetryEvents.ts new file mode 100644 index 000000000..e020c94e6 --- /dev/null +++ b/src/telemetry/telemetryEvents.ts @@ -0,0 +1,666 @@ +import type { ExtensionCommand } from '../commands'; +import type { DocumentSourceDetails } from '../documentSource'; +import { DocumentSource } from '../documentSource'; +import type { + ExportToPlaygroundError, + ParticipantErrorTypes, +} from '../participant/participantErrorTypes'; +import type { + ParticipantCommandType, + ParticipantRequestType, + ParticipantResponseType, +} from '../participant/participantTypes'; +import type { ShellEvaluateResult } from '../types/playgroundType'; +import type { NewConnectionTelemetryEventProperties } from './connectionTelemetry'; +import * as vscode from 'vscode'; + +type PlaygroundFileType = 'other' | 'mongodbjs' | 'mongodb'; + +type TelemetryFeedbackKind = 'positive' | 'negative' | undefined; + +export type InternalPromptPurpose = 'intent' | 'namespace' | undefined; + +export type ParticipantTelemetryMetadata = { + // The source of the participant prompt - e.g. 'codelens', 'treeview', etc. + source: DocumentSource; + + // Additional details about the source - e.g. if it's 'treeview', the detail can be 'database' or 'collection'. + source_details: DocumentSourceDetails; +}; + +export type ParticipantPromptProperties = { + command: ParticipantCommandType; + userInputLength: number; + totalMessageLength: number; + hasSampleDocuments: boolean; + historySize: number; + internalPurpose: InternalPromptPurpose; +}; + +function getPlaygroundFileTypeFromUri( + fileUri?: vscode.Uri +): PlaygroundFileType { + let fileType: PlaygroundFileType = 'other'; + + if (fileUri?.fsPath.match(/\.(mongodb\.js)$/gi)) { + fileType = 'mongodbjs'; + } else if (fileUri?.fsPath.match(/\.(mongodb)$/gi)) { + fileType = 'mongodb'; + } + + return fileType; +} + +type PlaygrdoundType = + | 'search' + | 'createCollection' + | 'createDatabase' + | 'index' + | 'agent' + | 'cloneDocument' + | 'insertDocument' + | 'createStreamProcessor' + | 'fromDatabaseTreeItem' + | 'fromCollectionTreeItem' + | 'crud'; + +type TelemetryEventType = + | 'Playground Code Executed' + | 'Link Clicked' + | 'Command Run' + | 'New Connection' + | 'Connection Edited' + | 'Open Edit Connection' + | 'Playground Saved' + | 'Playground Loaded' + | 'Document Updated' + | 'Document Edited' + | 'Playground Exported To Language' + | 'Playground Created' + | 'Export To Playground Failed' + | 'Saved Connections Loaded' + | 'Participant Feedback' + | 'Participant Welcome Shown' + | 'Participant Response Failed' + /** Tracks all submitted prompts */ + | 'Participant Prompt Submitted' + /** Tracks prompts that were submitted as a result of an action other than + * the user typing the message, such as clicking on an item in tree view or a codelens */ + | 'Participant Prompt Submitted From Action' + /** Tracks when a new chat was opened from an action such as clicking on a tree view. */ + | 'Participant Chat Opened From Action' + /** Tracks after a participant interacts with the input box we open to let the user write the prompt for participant. */ + | 'Participant Inbox Box Submitted' + | 'Participant Response Generated' + | 'Preset Connection Edited'; + +abstract class TelemetryEventBase { + abstract type: TelemetryEventType; + abstract properties: Record; +} + +// Reported when a playground file is run +export class PlaygroundExecutedTelemetryEvent implements TelemetryEventBase { + type: TelemetryEventType = 'Playground Code Executed'; + properties: { + // The type of the executed operation, e.g. 'insert', 'update', 'delete', 'query', 'aggregation', 'other' + type: string | null; + + // Whether the entire script was run or just a part of it + partial: boolean; + + // Whether an error occurred during execution + error: boolean; + }; + + constructor(result: ShellEvaluateResult, partial: boolean, error: boolean) { + this.properties = { + type: result ? this.getPlaygroundResultType(result) : null, + partial, + error, + }; + } + + private getPlaygroundResultType(res: ShellEvaluateResult): string { + if (!res || !res.result || !res.result.type) { + return 'other'; + } + + const shellApiType = res.result.type.toLocaleLowerCase(); + + // See: https://github.com/mongodb-js/mongosh/blob/main/packages/shell-api/src/shell-api.ts + if (shellApiType.includes('insert')) { + return 'insert'; + } + if (shellApiType.includes('update')) { + return 'update'; + } + if (shellApiType.includes('delete')) { + return 'delete'; + } + if (shellApiType.includes('aggregation')) { + return 'aggregation'; + } + if (shellApiType.includes('cursor')) { + return 'query'; + } + + return 'other'; + } +} + +// Reported when a user clicks a hyperlink - e.g. from the Help pane +export class LinkClickedTelemetryEvent implements TelemetryEventBase { + type: TelemetryEventType = 'Link Clicked'; + properties: { + // The screen where the link was clicked + screen: string; + + // The ID of the clicked link - e.g. `whatsNew`, `extensionDocumentation`, etc. + link_id: string; + }; + + constructor(screen: string, linkId: string) { + this.properties = { screen, link_id: linkId }; + } +} + +// Reported when any command is run by the user. Commands are the building blocks +// of the extension and can be executed either by clicking a UI element or by opening +// the command pallette (CMD+Shift+P). This event is likely to duplicate other events +// as it's fired automatically, regardless of other more-specific events. +export class CommandRunTelemetryEvent implements TelemetryEventBase { + type: TelemetryEventType = 'Command Run'; + properties: { + // The command that was executed - e.g. `mdb.connect`, `mdb.openMongoDBIssueReporter`, etc. + command: ExtensionCommand; + }; + + constructor(command: ExtensionCommand) { + this.properties = { command }; + } +} + +// Reported every time we connect to a cluster/db +export class NewConnectionTelemetryEvent implements TelemetryEventBase { + type: TelemetryEventType = 'New Connection'; + properties: NewConnectionTelemetryEventProperties; + + constructor(properties: NewConnectionTelemetryEventProperties) { + this.properties = properties; + } +} + +// Reported when a connection is edited +export class ConnectionEditedTelemetryEvent implements TelemetryEventBase { + type: TelemetryEventType = 'Connection Edited'; + properties = {}; +} + +// Reported when the user opens the connection editor +export class OpenEditConnectionTelemetryEvent implements TelemetryEventBase { + type: TelemetryEventType = 'Open Edit Connection'; + properties = {}; +} + +// Reported when a playground file is saved +export class PlaygroundSavedTelemetryEvent implements TelemetryEventBase { + type: TelemetryEventType = 'Playground Saved'; + properties: { + // The type of the file, e.g. 'mongodbjs' for .mongodb.js or 'mongodb' for .mongodb + file_type: PlaygroundFileType; + }; + + constructor(fileUri?: vscode.Uri) { + this.properties = { file_type: getPlaygroundFileTypeFromUri(fileUri) }; + } +} + +// Reported when a playground file is opened +export class PlaygroundLoadedTelemetryEvent implements TelemetryEventBase { + type: TelemetryEventType = 'Playground Loaded'; + properties: { + // The type of the file, e.g. 'mongodbjs' for .mongodb.js or 'mongodb' for .mongodb + file_type: PlaygroundFileType; + }; + + constructor(fileUri?: vscode.Uri) { + this.properties = { file_type: getPlaygroundFileTypeFromUri(fileUri) }; + } +} + +// Reported when a document is saved (e.g. when the user edits a document from a collection) +export class DocumentUpdatedTelemetryEvent implements TelemetryEventBase { + type: TelemetryEventType = 'Document Updated'; + properties: { + // The source of the document update, e.g. 'editor', 'tree_view', etc. + source: DocumentSource; + + // Whether the operation was successful + success: boolean; + }; + + constructor(source: DocumentSource, success: boolean) { + this.properties = { source, success }; + } +} + +// Reported when a document is opened in the editor, e.g. from a query results view +export class DocumentEditedTelemetryEvent implements TelemetryEventBase { + type: TelemetryEventType = 'Document Edited'; + properties: { + // The source of the document - e.g. codelens, treeview, etc. + source: DocumentSource; + }; + + constructor(source: DocumentSource) { + this.properties = { source }; + } +} + +// Reported when a playground file is exported to a language +export class PlaygroundExportedToLanguageTelemetryEvent + implements TelemetryEventBase +{ + type: TelemetryEventType = 'Playground Exported To Language'; + properties: { + // The target language of the export + language: string; + + // The length of the exported code + exported_code_length: number; + + // Whether the user opted to include driver syntax (e.g. import statements) + with_driver_syntax: boolean; + }; + + constructor( + language: string, + exportedCodeLength: number | undefined, + withDriverSyntax: boolean + ) { + this.properties = { + language, + exported_code_length: exportedCodeLength || 0, + with_driver_syntax: withDriverSyntax, + }; + } +} + +// Reported when a new playground is created +export class PlaygroundCreatedTelemetryEvent implements TelemetryEventBase { + type: TelemetryEventType = 'Playground Created'; + properties: { + // The playground type - e.g. 'search', 'createCollection', 'createDatabase', etc. This is typically + // indicative of the element the user clicked to create the playground. + playground_type: PlaygrdoundType; + }; + + constructor(playgroundType: PlaygrdoundType) { + this.properties = { playground_type: playgroundType }; + } +} +// Reported when exporting to playground fails +export class ExportToPlaygroundFailedTelemetryEvent + implements TelemetryEventBase +{ + type: TelemetryEventType = 'Export To Playground Failed'; + properties: { + // The length of the playground code + input_length: number | undefined; + + // The name of the error that occurred + error_name?: ExportToPlaygroundError; + }; + + constructor( + inputLength: number | undefined, + errorName: ExportToPlaygroundError + ) { + this.properties = { input_length: inputLength, error_name: errorName }; + } +} + +// Reported when saved connections are loaded from disk. This is currently disabled +// due to the large volume of events. +export class SavedConnectionsLoadedTelemetryEvent + implements TelemetryEventBase +{ + type: TelemetryEventType = 'Saved Connections Loaded'; + properties: { + // Total number of connections saved on disk + saved_connections: number; + + // Total number of connections from preset settings + preset_connections: number; + + // Total number of connections that extension was able to load, it might + // differ from saved_connections since there might be failures in loading + // secrets for a connection in which case we don't list the connections in the + // list of loaded connections. + loaded_connections: number; + + // Total number of connections that have secrets stored in keytar + connections_with_secrets_in_keytar: number; + + // Total number of connections that have secrets stored in secret storage + connections_with_secrets_in_secret_storage: number; + }; + + constructor({ + savedConnections, + presetConnections, + loadedConnections, + connectionsWithSecretsInKeytar, + connectionsWithSecretsInSecretStorage, + }: { + savedConnections: number; + presetConnections: number; + loadedConnections: number; + connectionsWithSecretsInKeytar: number; + connectionsWithSecretsInSecretStorage: number; + }) { + this.properties = { + saved_connections: savedConnections, + preset_connections: presetConnections, + loaded_connections: loadedConnections, + connections_with_secrets_in_keytar: connectionsWithSecretsInKeytar, + connections_with_secrets_in_secret_storage: + connectionsWithSecretsInSecretStorage, + }; + } +} + +// Reported when the user provides feedback to the chatbot on a response +export class ParticipantFeedbackTelemetryEvent implements TelemetryEventBase { + type: TelemetryEventType = 'Participant Feedback'; + properties: { + // The type of feedback provided - e.g. 'positive', 'negative' + feedback: TelemetryFeedbackKind; + + // The response type that the feedback was provided for - e.g. 'query', 'schema', 'docs' + response_type: ParticipantResponseType; + + // If the feedback was negative, the reason for the negative feedback. It's picked from + // a set of predefined options and not a free-form text field. + reason?: String; + }; + + constructor( + feedback: vscode.ChatResultFeedbackKind, + responseType: ParticipantResponseType, + reason?: String + ) { + this.properties = { + feedback: this.chatResultFeedbackKindToTelemetryValue(feedback), + response_type: responseType, + reason, + }; + } + + private chatResultFeedbackKindToTelemetryValue( + kind: vscode.ChatResultFeedbackKind + ): TelemetryFeedbackKind { + switch (kind) { + case vscode.ChatResultFeedbackKind.Helpful: + return 'positive'; + case vscode.ChatResultFeedbackKind.Unhelpful: + return 'negative'; + default: + return undefined; + } + } +} + +// Reported when the participant welcome message is shown +export class ParticipantWelcomeShownTelemetryEvent + implements TelemetryEventBase +{ + type: TelemetryEventType = 'Participant Welcome Shown'; + properties = {}; +} + +// Reported when a participant response fails +export class ParticipantResponseFailedTelemetryEvent + implements TelemetryEventBase +{ + type: TelemetryEventType = 'Participant Response Failed'; + properties: { + // The type of the command that failed - e.g. 'query', 'schema', 'docs' + command: ParticipantResponseType; + + // The error code that caused the failure + error_code?: string; + + // The name of the error that caused the failure + error_name: ParticipantErrorTypes; + + // Additional details about the error if any. + error_details?: string; + }; + + constructor( + command: ParticipantResponseType, + errorName: ParticipantErrorTypes, + errorCode?: string, + errorDetails?: string + ) { + this.properties = { + command, + error_code: errorCode, + error_name: errorName, + error_details: errorDetails, + }; + } +} + +// Reported when a participant prompt is submitted +export class ParticipantPromptSubmittedTelemetryEvent + implements TelemetryEventBase +{ + type: TelemetryEventType = 'Participant Prompt Submitted'; + properties: { + // The type of the command that was submitted - e.g. 'query', 'schema', 'docs' + command: ParticipantCommandType; + + // The length of the user input + user_input_length: number; + + // The total length of the message - i.e. user input + participant prompt + total_message_length: number; + + // Whether the prompt has sample documents + has_sample_documents: boolean; + + // The size of the history + history_size: number; + + // For internal prompts - e.g. trying to extract the 'intent', 'namespace' or the + // namespace from the chat history. + internal_purpose: InternalPromptPurpose; + }; + + constructor({ + command, + userInputLength, + totalMessageLength, + hasSampleDocuments, + historySize, + internalPurpose, + }: ParticipantPromptProperties) { + this.properties = { + command: command, + user_input_length: userInputLength, + total_message_length: totalMessageLength, + has_sample_documents: hasSampleDocuments, + history_size: historySize, + internal_purpose: internalPurpose, + }; + } +} + +// Reported when a participant prompt is submitted from an action other than typing directly. +// This is typically one of the activation points - e.g. clicking on the tree view, a codelens, etc. +export class ParticipantPromptSubmittedFromActionTelemetryEvent + implements TelemetryEventBase +{ + type: TelemetryEventType = 'Participant Prompt Submitted From Action'; + properties: ParticipantTelemetryMetadata & { + // The length of the input + input_length: number; + + // The command we're requesting - e.g. 'query', 'schema', 'docs' + command: ParticipantRequestType; + }; + + constructor( + sourceMetadata: ParticipantTelemetryMetadata, + requestType: ParticipantRequestType, + inputLength: number + ) { + this.properties = { + ...sourceMetadata, + input_length: inputLength, + command: requestType, + }; + } +} + +// Reported when a new chat is initiated from an activation point in the extension (e.g. the database tree view) +export class ParticipantChatOpenedFromActionTelemetryEvent + implements TelemetryEventBase +{ + type: TelemetryEventType = 'Participant Chat Opened From Action'; + properties: ParticipantTelemetryMetadata & { + // The command - if any - we're opening a chat for - e.g. 'query', 'schema', 'docs' + command?: ParticipantCommandType; + }; + + constructor( + sourceMetadata: ParticipantTelemetryMetadata, + command?: ParticipantCommandType + ) { + this.properties = { ...sourceMetadata, command }; + } +} + +// Reported when we open an input box to ask the user for a message that we'll send to copilot +export class ParticipantInputBoxSubmittedTelemetryEvent + implements TelemetryEventBase +{ + type: TelemetryEventType = 'Participant Inbox Box Submitted'; + properties: ParticipantTelemetryMetadata & { + // The supplied input length + input_length: number; + + // Whether the input was dismissed + dismissed: boolean; + + // The command we're requesting - e.g. 'query', 'schema', 'docs' + command?: ParticipantCommandType; + }; + + constructor( + sourceMetadata: ParticipantTelemetryMetadata, + message: string | undefined, + command?: ParticipantCommandType + ) { + this.properties = { + ...sourceMetadata, + input_length: message?.length || 0, + dismissed: message === undefined, + command, + }; + } +} + +// Reported when a participant response is generated +export class ParticipantResponseGeneratedTelemetryEvent + implements TelemetryEventBase +{ + type: TelemetryEventType = 'Participant Response Generated'; + properties: { + // The type of the command that was requested - e.g. 'query', 'schema', 'docs' + command: ParticipantResponseType; + + // Whether the response has a call to action (e.g. 'Open in playground' button) + has_cta: boolean; + + // Whether the response has runnable content (e.g. a code block) + has_runnable_content: boolean; + + // Whether the response contains namespace information + found_namespace: boolean; + + // The length of the output + output_length: number; + }; + + constructor({ + command, + hasCta, + hasRunnableContent, + foundNamespace, + outputLength, + }: { + command: ParticipantResponseType; + hasCta: boolean; + hasRunnableContent: boolean; + foundNamespace: boolean; + outputLength: number; + }) { + this.properties = { + command, + has_cta: hasCta, + has_runnable_content: hasRunnableContent, + found_namespace: foundNamespace, + output_length: outputLength, + }; + } +} + +// Reported when a preset connection is edited +export class PresetConnectionEditedTelemetryEvent + implements TelemetryEventBase +{ + type: TelemetryEventType = 'Preset Connection Edited'; + properties: { + // The source of the interaction - currently, only treeview + source: Extract; + + // Additional details about the source - e.g. if it's a specific connection element, + // it'll be 'tree_item', otherwise it'll be 'header'. + source_details: 'tree_item' | 'header'; + }; + + constructor(sourceDetails: 'tree_item' | 'header') { + this.properties = { + source: DocumentSource.DOCUMENT_SOURCE_TREEVIEW, + source_details: sourceDetails, + }; + } +} + +export type TelemetryEvent = + | PlaygroundExecutedTelemetryEvent + | LinkClickedTelemetryEvent + | CommandRunTelemetryEvent + | NewConnectionTelemetryEvent + | ConnectionEditedTelemetryEvent + | OpenEditConnectionTelemetryEvent + | PlaygroundSavedTelemetryEvent + | PlaygroundLoadedTelemetryEvent + | DocumentUpdatedTelemetryEvent + | DocumentEditedTelemetryEvent + | PlaygroundExportedToLanguageTelemetryEvent + | PlaygroundCreatedTelemetryEvent + | ExportToPlaygroundFailedTelemetryEvent + | SavedConnectionsLoadedTelemetryEvent + | ParticipantFeedbackTelemetryEvent + | ParticipantWelcomeShownTelemetryEvent + | ParticipantPromptSubmittedTelemetryEvent + | ParticipantPromptSubmittedFromActionTelemetryEvent + | ParticipantChatOpenedFromActionTelemetryEvent + | ParticipantInputBoxSubmittedTelemetryEvent + | ParticipantResponseGeneratedTelemetryEvent + | PresetConnectionEditedTelemetryEvent; diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts index 82ccd523b..5ce237ce3 100644 --- a/src/telemetry/telemetryService.ts +++ b/src/telemetry/telemetryService.ts @@ -7,225 +7,26 @@ import { Analytics as SegmentAnalytics } from '@segment/analytics-node'; import type { ConnectionTypes } from '../connectionController'; import { createLogger } from '../logging'; -import type { DocumentSource } from '../documentSource'; import { getConnectionTelemetryProperties } from './connectionTelemetry'; -import type { NewConnectionTelemetryEventProperties } from './connectionTelemetry'; -import type { ShellEvaluateResult } from '../types/playgroundType'; import type { StorageController } from '../storage'; -import type { ExportToPlaygroundError } from '../participant/participantErrorTypes'; import { ParticipantErrorTypes } from '../participant/participantErrorTypes'; -import type { ExtensionCommand } from '../commands'; -import type { - ParticipantCommandType, - ParticipantRequestType, - ParticipantResponseType, -} from '../participant/participantTypes'; +import type { ParticipantResponseType } from '../participant/participantTypes'; +import type { TelemetryEvent } from './telemetryEvents'; +import { + NewConnectionTelemetryEvent, + ParticipantResponseFailedTelemetryEvent, +} from './telemetryEvents'; const log = createLogger('telemetry'); // eslint-disable-next-line @typescript-eslint/no-var-requires const { version } = require('../../package.json'); -type PlaygroundTelemetryEventProperties = { - type: string | null; - partial: boolean; - error: boolean; -}; - export type SegmentProperties = { event: string; anonymousId: string; properties: Record; }; -type LinkClickedTelemetryEventProperties = { - screen: string; - link_id: string; -}; - -type ExtensionCommandRunTelemetryEventProperties = { - command: ExtensionCommand; -}; - -type DocumentUpdatedTelemetryEventProperties = { - source: DocumentSource; - success: boolean; -}; - -type DocumentEditedTelemetryEventProperties = { - source: DocumentSource; -}; - -type ExportToPlaygroundFailedEventProperties = { - input_length: number | undefined; - error_name?: ExportToPlaygroundError; -}; - -type PlaygroundExportedToLanguageTelemetryEventProperties = { - language?: string; - exported_code_length: number; - with_driver_syntax?: boolean; -}; - -type PlaygroundCreatedTelemetryEventProperties = { - playground_type: string; -}; - -type PlaygroundSavedTelemetryEventProperties = { - file_type?: string; -}; - -type PlaygroundLoadedTelemetryEventProperties = { - file_type?: string; -}; - -type KeytarSecretsMigrationFailedProperties = { - saved_connections: number; - loaded_connections: number; - connections_with_failed_keytar_migration: number; -}; - -type ConnectionEditedTelemetryEventProperties = { - success: boolean; -}; - -type SavedConnectionsLoadedProperties = { - // Total number of connections saved on disk - saved_connections: number; - // Total number of connections from preset settings - preset_connections: number; - // Total number of connections that extension was able to load, it might - // differ from saved_connections since there might be failures in loading - // secrets for a connection in which case we don't list the connections in the - // list of loaded connections. - loaded_connections: number; - connections_with_secrets_in_keytar: number; - connections_with_secrets_in_secret_storage: number; -}; - -type TelemetryFeedbackKind = 'positive' | 'negative' | undefined; - -type ParticipantFeedbackProperties = { - feedback: TelemetryFeedbackKind; - response_type: ParticipantResponseType; - reason?: String; -}; - -type ParticipantResponseFailedProperties = { - command: ParticipantResponseType; - error_code?: string; - error_name: ParticipantErrorTypes; - error_details?: string; -}; - -export type InternalPromptPurpose = 'intent' | 'namespace' | undefined; - -export type ParticipantPromptProperties = { - command: ParticipantCommandType; - user_input_length: number; - total_message_length: number; - has_sample_documents: boolean; - history_size: number; - internal_purpose: InternalPromptPurpose; -}; - -export type ParticipantResponseProperties = { - command: ParticipantResponseType; - has_cta: boolean; - has_runnable_content: boolean; - found_namespace: boolean; - output_length: number; -}; - -export type ParticipantPromptSubmittedFromActionProperties = { - source: DocumentSource; - input_length: number; - command: ParticipantRequestType; -}; - -export type ParticipantChatOpenedFromActionProperties = { - source: DocumentSource; - command?: ParticipantCommandType; -}; - -export type PresetSavedConnectionEditedProperties = { - source: DocumentSource; - source_details: 'tree_item' | 'header'; -}; - -export type ParticipantInputBoxSubmitted = { - source: DocumentSource; - input_length: number | undefined; - dismissed: boolean; - command?: ParticipantCommandType; -}; - -export function chatResultFeedbackKindToTelemetryValue( - kind: vscode.ChatResultFeedbackKind -): TelemetryFeedbackKind { - switch (kind) { - case vscode.ChatResultFeedbackKind.Helpful: - return 'positive'; - case vscode.ChatResultFeedbackKind.Unhelpful: - return 'negative'; - default: - return undefined; - } -} - -type TelemetryEventProperties = - | PlaygroundTelemetryEventProperties - | LinkClickedTelemetryEventProperties - | ExtensionCommandRunTelemetryEventProperties - | NewConnectionTelemetryEventProperties - | DocumentUpdatedTelemetryEventProperties - | ConnectionEditedTelemetryEventProperties - | DocumentEditedTelemetryEventProperties - | PlaygroundExportedToLanguageTelemetryEventProperties - | PlaygroundCreatedTelemetryEventProperties - | PlaygroundSavedTelemetryEventProperties - | PlaygroundLoadedTelemetryEventProperties - | KeytarSecretsMigrationFailedProperties - | ExportToPlaygroundFailedEventProperties - | SavedConnectionsLoadedProperties - | ParticipantFeedbackProperties - | ParticipantResponseFailedProperties - | ParticipantPromptProperties - | ParticipantPromptSubmittedFromActionProperties - | ParticipantChatOpenedFromActionProperties - | ParticipantResponseProperties; - -export enum TelemetryEventTypes { - PLAYGROUND_CODE_EXECUTED = 'Playground Code Executed', - EXTENSION_LINK_CLICKED = 'Link Clicked', - EXTENSION_COMMAND_RUN = 'Command Run', - NEW_CONNECTION = 'New Connection', - CONNECTION_EDITED = 'Connection Edited', - OPEN_EDIT_CONNECTION = 'Open Edit Connection', - PLAYGROUND_SAVED = 'Playground Saved', - PLAYGROUND_LOADED = 'Playground Loaded', - DOCUMENT_UPDATED = 'Document Updated', - DOCUMENT_EDITED = 'Document Edited', - PLAYGROUND_EXPORTED_TO_LANGUAGE = 'Playground Exported To Language', - PLAYGROUND_CREATED = 'Playground Created', - KEYTAR_SECRETS_MIGRATION_FAILED = 'Keytar Secrets Migration Failed', - EXPORT_TO_PLAYGROUND_FAILED = 'Export To Playground Failed', - SAVED_CONNECTIONS_LOADED = 'Saved Connections Loaded', - PARTICIPANT_FEEDBACK = 'Participant Feedback', - PARTICIPANT_WELCOME_SHOWN = 'Participant Welcome Shown', - PARTICIPANT_RESPONSE_FAILED = 'Participant Response Failed', - /** Tracks all submitted prompts */ - PARTICIPANT_PROMPT_SUBMITTED = 'Participant Prompt Submitted', - /** Tracks prompts that were submitted as a result of an action other than - * the user typing the message, such as clicking on an item in tree view or a codelens */ - PARTICIPANT_PROMPT_SUBMITTED_FROM_ACTION = 'Participant Prompt Submitted From Action', - /** Tracks when a new chat was opened from an action such as clicking on a tree view. */ - PARTICIPANT_CHAT_OPENED_FROM_ACTION = 'Participant Chat Opened From Action', - /** Tracks after a participant interacts with the input box we open to let the user write the prompt for participant. */ - PARTICIPANT_INPUT_BOX_SUBMITTED = 'Participant Inbox Box Submitted', - PARTICIPANT_RESPONSE_GENERATED = 'Participant Response Generated', - PRESET_CONNECTION_EDITED = 'Preset Connection Edited', -} - /** * This controller manages telemetry. */ @@ -325,16 +126,13 @@ export default class TelemetryService { }); } - track( - eventType: TelemetryEventTypes, - properties?: TelemetryEventProperties - ): void { + track(event: TelemetryEvent): void { try { this._segmentAnalyticsTrack({ ...this.getTelemetryUserIdentity(), - event: eventType, + event: event.type, properties: { - ...properties, + ...event.properties, extension_version: `${version}`, }, }); @@ -343,61 +141,14 @@ export default class TelemetryService { } } - async _getConnectionTelemetryProperties( - dataService: DataService, - connectionType: ConnectionTypes - ): Promise { - return await getConnectionTelemetryProperties(dataService, connectionType); - } - async trackNewConnection( dataService: DataService, connectionType: ConnectionTypes ): Promise { const connectionTelemetryProperties = - await this._getConnectionTelemetryProperties(dataService, connectionType); - - this.track( - TelemetryEventTypes.NEW_CONNECTION, - connectionTelemetryProperties - ); - } - - trackExportToPlaygroundFailed( - props: ExportToPlaygroundFailedEventProperties - ): void { - this.track(TelemetryEventTypes.EXPORT_TO_PLAYGROUND_FAILED, props); - } + await getConnectionTelemetryProperties(dataService, connectionType); - trackCommandRun(command: ExtensionCommand): void { - this.track(TelemetryEventTypes.EXTENSION_COMMAND_RUN, { command }); - } - - getPlaygroundResultType(res: ShellEvaluateResult): string { - if (!res || !res.result || !res.result.type) { - return 'other'; - } - - const shellApiType = res.result.type.toLocaleLowerCase(); - - // See: https://github.com/mongodb-js/mongosh/blob/main/packages/shell-api/src/shell-api.js - if (shellApiType.includes('insert')) { - return 'insert'; - } - if (shellApiType.includes('update')) { - return 'update'; - } - if (shellApiType.includes('delete')) { - return 'delete'; - } - if (shellApiType.includes('aggregation')) { - return 'aggregation'; - } - if (shellApiType.includes('cursor')) { - return 'query'; - } - - return 'other'; + this.track(new NewConnectionTelemetryEvent(connectionTelemetryProperties)); } getTelemetryUserIdentity(): { anonymousId: string } { @@ -406,107 +157,6 @@ export default class TelemetryService { }; } - trackPlaygroundCodeExecuted( - result: ShellEvaluateResult, - partial: boolean, - error: boolean - ): void { - this.track(TelemetryEventTypes.PLAYGROUND_CODE_EXECUTED, { - type: result ? this.getPlaygroundResultType(result) : null, - partial, - error, - }); - } - - trackLinkClicked(screen: string, linkId: string): void { - this.track(TelemetryEventTypes.EXTENSION_LINK_CLICKED, { - screen, - link_id: linkId, - }); - } - - trackPlaygroundLoaded(fileType?: string): void { - this.track(TelemetryEventTypes.PLAYGROUND_LOADED, { - file_type: fileType, - }); - } - - trackPlaygroundSaved(fileType?: string): void { - this.track(TelemetryEventTypes.PLAYGROUND_SAVED, { - file_type: fileType, - }); - } - - trackDocumentUpdated(source: DocumentSource, success: boolean): void { - this.track(TelemetryEventTypes.DOCUMENT_UPDATED, { source, success }); - } - - trackDocumentOpenedInEditor(source: DocumentSource): void { - this.track(TelemetryEventTypes.DOCUMENT_EDITED, { source }); - } - - trackPlaygroundExportedToLanguageExported( - playgroundExportedProps: PlaygroundExportedToLanguageTelemetryEventProperties - ): void { - this.track( - TelemetryEventTypes.PLAYGROUND_EXPORTED_TO_LANGUAGE, - playgroundExportedProps - ); - } - - trackPresetConnectionEdited( - props: PresetSavedConnectionEditedProperties - ): void { - this.track(TelemetryEventTypes.PRESET_CONNECTION_EDITED, props); - } - - trackPlaygroundCreated(playgroundType: string): void { - this.track(TelemetryEventTypes.PLAYGROUND_CREATED, { - playground_type: playgroundType, - }); - } - - trackSavedConnectionsLoaded( - savedConnectionsLoadedProps: SavedConnectionsLoadedProperties - ): void { - this.track( - TelemetryEventTypes.SAVED_CONNECTIONS_LOADED, - savedConnectionsLoadedProps - ); - } - - trackKeytarSecretsMigrationFailed( - keytarSecretsMigrationFailedProps: KeytarSecretsMigrationFailedProperties - ): void { - this.track( - TelemetryEventTypes.KEYTAR_SECRETS_MIGRATION_FAILED, - keytarSecretsMigrationFailedProps - ); - } - - trackParticipantFeedback(props: ParticipantFeedbackProperties): void { - this.track(TelemetryEventTypes.PARTICIPANT_FEEDBACK, props); - } - - trackParticipantPromptSubmittedFromAction( - props: ParticipantPromptSubmittedFromActionProperties - ): void { - this.track( - TelemetryEventTypes.PARTICIPANT_PROMPT_SUBMITTED_FROM_ACTION, - props - ); - } - - trackParticipantChatOpenedFromAction( - props: ParticipantChatOpenedFromActionProperties - ): void { - this.track(TelemetryEventTypes.PARTICIPANT_CHAT_OPENED_FROM_ACTION, props); - } - - trackParticipantInputBoxSubmitted(props: ParticipantInputBoxSubmitted): void { - this.track(TelemetryEventTypes.PARTICIPANT_INPUT_BOX_SUBMITTED, props); - } - trackParticipantError(err: any, command: ParticipantResponseType): void { let errorCode: string | undefined; let errorName: ParticipantErrorTypes; @@ -535,18 +185,8 @@ export default class TelemetryService { errorName = ParticipantErrorTypes.OTHER; } - this.track(TelemetryEventTypes.PARTICIPANT_RESPONSE_FAILED, { - command, - error_code: errorCode, - error_name: errorName, - } satisfies ParticipantResponseFailedProperties); - } - - trackParticipantPrompt(stats: ParticipantPromptProperties): void { - this.track(TelemetryEventTypes.PARTICIPANT_PROMPT_SUBMITTED, stats); - } - - trackParticipantResponse(props: ParticipantResponseProperties): void { - this.track(TelemetryEventTypes.PARTICIPANT_RESPONSE_GENERATED, props); + this.track( + new ParticipantResponseFailedTelemetryEvent(command, errorName, errorCode) + ); } } diff --git a/src/test/suite/connectionController.test.ts b/src/test/suite/connectionController.test.ts index 0f54d5820..943a0950f 100644 --- a/src/test/suite/connectionController.test.ts +++ b/src/test/suite/connectionController.test.ts @@ -19,7 +19,7 @@ import { SecretStorageLocation, } from '../../storage/storageController'; import { StatusView } from '../../views'; -import TelemetryService from '../../telemetry/telemetryService'; +import TelemetryService from '../../telemetry'; import { ExtensionContextStub } from './stubs'; import { TEST_DATABASE_URI, @@ -818,7 +818,7 @@ suite('Connection Controller Test Suite', function () { test('two disconnects on one connection at once complete without erroring', (done) => { let disconnectsCompleted = 0; - async function disconnect() { + async function disconnect(): Promise { try { await testConnectionController.disconnect(); @@ -1213,10 +1213,7 @@ suite('Connection Controller Test Suite', function () { '_getConnectionInfoWithSecrets', (connectionInfo) => Promise.resolve(connectionInfo as LoadedConnection) ); - const trackStub = testSandbox.stub( - testTelemetryService, - 'trackSavedConnectionsLoaded' - ); + const trackStub = testSandbox.stub(testTelemetryService, 'track'); // Clear any connections and load so we get our stubbed connections from above. testConnectionController.clearAllConnections(); diff --git a/src/test/suite/editors/activeConnectionCodeLensProvider.test.ts b/src/test/suite/editors/activeConnectionCodeLensProvider.test.ts index 05a65ca53..62f7a5708 100644 --- a/src/test/suite/editors/activeConnectionCodeLensProvider.test.ts +++ b/src/test/suite/editors/activeConnectionCodeLensProvider.test.ts @@ -10,7 +10,7 @@ import ConnectionController from '../../../connectionController'; import { StatusView } from '../../../views'; import { StorageController } from '../../../storage'; import { ExtensionContextStub } from '../stubs'; -import TelemetryService from '../../../telemetry/telemetryService'; +import TelemetryService from '../../../telemetry'; import { TEST_DATABASE_URI } from '../dbTestHelper'; suite('Active Connection CodeLens Provider Test Suite', () => { diff --git a/src/test/suite/editors/collectionDocumentsProvider.test.ts b/src/test/suite/editors/collectionDocumentsProvider.test.ts index eb844baa7..f23aadd74 100644 --- a/src/test/suite/editors/collectionDocumentsProvider.test.ts +++ b/src/test/suite/editors/collectionDocumentsProvider.test.ts @@ -17,9 +17,10 @@ import { SecretStorageLocation, StorageLocation, } from '../../../storage/storageController'; -import TelemetryService from '../../../telemetry/telemetryService'; +import TelemetryService from '../../../telemetry'; import { TEST_DATABASE_URI } from '../dbTestHelper'; import { ExtensionContextStub, mockTextEditor } from '../stubs'; +import { expect } from 'chai'; const mockDocumentsAsJsonString = `[ { @@ -172,6 +173,7 @@ suite('Collection Documents Provider Test Suite', () => { sandbox.stub(testCollectionViewProvider._statusView, 'hideMessage'); await testCollectionViewProvider.provideTextDocumentContent(uri); + assert( testQueryStore.operations[operationId].hasMoreDocumentsToShow === false, 'Expected not to have more documents to show.' @@ -202,7 +204,7 @@ suite('Collection Documents Provider Test Suite', () => { const showMessageStub = sandbox.stub(testStatusView, 'showMessage'); const hideMessageStub = sandbox.stub(testStatusView, 'hideMessage'); - mockActiveDataService.find = () => { + mockActiveDataService.find = (): Promise<{ field: string }[]> => { assert(showMessageStub.called); assert(!hideMessageStub.called); assert(showMessageStub.firstCall.args[0] === 'Fetching documents...'); diff --git a/src/test/suite/editors/editDocumentCodeLensProvider.test.ts b/src/test/suite/editors/editDocumentCodeLensProvider.test.ts index b553fa85a..b4f150303 100644 --- a/src/test/suite/editors/editDocumentCodeLensProvider.test.ts +++ b/src/test/suite/editors/editDocumentCodeLensProvider.test.ts @@ -11,7 +11,7 @@ import EditDocumentCodeLensProvider from '../../../editors/editDocumentCodeLensP import { mockTextEditor } from '../stubs'; import { StatusView } from '../../../views'; import { StorageController } from '../../../storage'; -import TelemetryService from '../../../telemetry/telemetryService'; +import TelemetryService from '../../../telemetry'; import { ExtensionContextStub } from '../stubs'; suite('Edit Document Code Lens Provider Test Suite', () => { diff --git a/src/test/suite/editors/exportToLanguageCodeLensProvider.test.ts b/src/test/suite/editors/exportToLanguageCodeLensProvider.test.ts index b0eb03545..268431f75 100644 --- a/src/test/suite/editors/exportToLanguageCodeLensProvider.test.ts +++ b/src/test/suite/editors/exportToLanguageCodeLensProvider.test.ts @@ -7,7 +7,7 @@ import ExportToLanguageCodeLensProvider, { import PlaygroundResultProvider from '../../../editors/playgroundResultProvider'; import StorageController from '../../../storage/storageController'; import { ExtensionContextStub } from '../stubs'; -import TelemetryService from '../../../telemetry/telemetryService'; +import TelemetryService from '../../../telemetry'; import StatusView from '../../../views/statusView'; import ConnectionController from '../../../connectionController'; import EditDocumentCodeLensProvider from '../../../editors/editDocumentCodeLensProvider'; diff --git a/src/test/suite/editors/mongoDBDocumentService.test.ts b/src/test/suite/editors/mongoDBDocumentService.test.ts index dafbd5829..5f9be2f86 100644 --- a/src/test/suite/editors/mongoDBDocumentService.test.ts +++ b/src/test/suite/editors/mongoDBDocumentService.test.ts @@ -11,7 +11,7 @@ import MongoDBDocumentService from '../../../editors/mongoDBDocumentService'; import { StorageController } from '../../../storage'; import { StatusView } from '../../../views'; -import TelemetryService from '../../../telemetry/telemetryService'; +import TelemetryService from '../../../telemetry'; import { ExtensionContextStub } from '../stubs'; const expect = chai.expect; diff --git a/src/test/suite/editors/playgroundController.test.ts b/src/test/suite/editors/playgroundController.test.ts index 0bb8cf545..eba4a2652 100644 --- a/src/test/suite/editors/playgroundController.test.ts +++ b/src/test/suite/editors/playgroundController.test.ts @@ -14,7 +14,7 @@ import { PlaygroundController } from '../../../editors'; import PlaygroundResultProvider from '../../../editors/playgroundResultProvider'; import { StatusView } from '../../../views'; import { StorageController } from '../../../storage'; -import TelemetryService from '../../../telemetry/telemetryService'; +import TelemetryService from '../../../telemetry'; import { TEST_DATABASE_URI } from '../dbTestHelper'; import { ExtensionContextStub, LanguageServerControllerStub } from '../stubs'; import { mockTextEditor } from '../stubs'; diff --git a/src/test/suite/editors/playgroundResultProvider.test.ts b/src/test/suite/editors/playgroundResultProvider.test.ts index b7d972d0f..25edda64f 100644 --- a/src/test/suite/editors/playgroundResultProvider.test.ts +++ b/src/test/suite/editors/playgroundResultProvider.test.ts @@ -14,7 +14,7 @@ import EditDocumentCodeLensProvider from '../../../editors/editDocumentCodeLensP import PlaygroundResultProvider from '../../../editors/playgroundResultProvider'; import { StatusView } from '../../../views'; import { StorageController } from '../../../storage'; -import TelemetryService from '../../../telemetry/telemetryService'; +import TelemetryService from '../../../telemetry'; import { ExtensionContextStub } from '../stubs'; const expect = chai.expect; diff --git a/src/test/suite/explorer/helpExplorer.test.ts b/src/test/suite/explorer/helpExplorer.test.ts index 9bb9df134..c86eae3c2 100644 --- a/src/test/suite/explorer/helpExplorer.test.ts +++ b/src/test/suite/explorer/helpExplorer.test.ts @@ -110,11 +110,11 @@ suite('Help Explorer Test Suite', function () { const testHelpExplorer = mdbTestExtension.testExtensionController._helpExplorer; - const stubLinkClickedTelemetry = sandbox.fake(); + const stubTrackTelemetry = sandbox.fake(); sandbox.replace( mdbTestExtension.testExtensionController._telemetryService, - 'trackLinkClicked', - stubLinkClickedTelemetry + 'track', + stubTrackTelemetry ); testHelpExplorer.activateHelpTreeView( mdbTestExtension.testExtensionController._telemetryService @@ -127,8 +127,13 @@ suite('Help Explorer Test Suite', function () { atlasHelpItem, mdbTestExtension.testExtensionController._telemetryService ); - assert(stubLinkClickedTelemetry.called); - assert(stubLinkClickedTelemetry.firstCall.args[0] === 'helpPanel'); - assert(stubLinkClickedTelemetry.firstCall.args[1] === 'freeClusterCTA'); + assert(stubTrackTelemetry.called); + assert( + stubTrackTelemetry.firstCall.args[0].properties.screen === 'helpPanel' + ); + assert( + stubTrackTelemetry.firstCall.args[0].properties.link_id === + 'freeClusterCTA' + ); }); }); diff --git a/src/test/suite/language/languageServerController.test.ts b/src/test/suite/language/languageServerController.test.ts index 686785729..a1b214791 100644 --- a/src/test/suite/language/languageServerController.test.ts +++ b/src/test/suite/language/languageServerController.test.ts @@ -18,7 +18,7 @@ import PlaygroundResultProvider from '../../../editors/playgroundResultProvider' import { StatusView } from '../../../views'; import { StorageController } from '../../../storage'; import { TEST_DATABASE_URI } from '../dbTestHelper'; -import TelemetryService from '../../../telemetry/telemetryService'; +import TelemetryService from '../../../telemetry'; import { ExtensionContextStub } from '../stubs'; import ExportToLanguageCodeLensProvider from '../../../editors/exportToLanguageCodeLensProvider'; diff --git a/src/test/suite/oidc.test.ts b/src/test/suite/oidc.test.ts index 41aa63099..cfcfb3579 100644 --- a/src/test/suite/oidc.test.ts +++ b/src/test/suite/oidc.test.ts @@ -11,7 +11,7 @@ import { before, after, afterEach, beforeEach } from 'mocha'; import EventEmitter, { once } from 'events'; import { ExtensionContextStub } from './stubs'; import { StorageController } from '../../storage'; -import { TelemetryService } from '../../telemetry'; +import TelemetryService from '../../telemetry'; import ConnectionController from '../../connectionController'; import { StatusView } from '../../views'; import { waitFor } from './waitFor'; diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts index 5d8565793..47539d7f1 100644 --- a/src/test/suite/participant/participant.test.ts +++ b/src/test/suite/participant/participant.test.ts @@ -12,13 +12,13 @@ import { StorageController } from '../../../storage'; import { StatusView } from '../../../views'; import { ExtensionContextStub } from '../stubs'; import type { + ExportToPlaygroundFailedTelemetryEvent, InternalPromptPurpose, - ParticipantPromptProperties, - ParticipantResponseProperties, -} from '../../../telemetry/telemetryService'; -import TelemetryService, { - TelemetryEventTypes, -} from '../../../telemetry/telemetryService'; + ParticipantFeedbackTelemetryEvent, + ParticipantPromptSubmittedTelemetryEvent, + ParticipantResponseFailedTelemetryEvent, + ParticipantResponseGeneratedTelemetryEvent, +} from '../../../telemetry'; import { TEST_DATABASE_URI } from '../dbTestHelper'; import type { ChatResult } from '../../../participant/constants'; import { CHAT_PARTICIPANT_ID } from '../../../participant/constants'; @@ -48,6 +48,7 @@ import type { SendMessageToParticipantOptions, } from '../../../participant/participantTypes'; import { DocumentSource } from '../../../documentSource'; +import TelemetryService from '../../../telemetry'; // The Copilot's model in not available in tests, // therefore we need to mock its methods and returning values. @@ -143,28 +144,29 @@ suite('Participant Controller Test Suite', function () { expect(telemetryTrackStub.callCount).to.be.greaterThan(callIndex); const call = telemetryTrackStub.getCalls()[callIndex]; - expect(call.args[0]).to.equal('Participant Prompt Submitted'); + const arg = call.args[0] as ParticipantPromptSubmittedTelemetryEvent; + expect(arg.type).to.equal('Participant Prompt Submitted'); - const properties = call.args[1] as ParticipantPromptProperties; - - expect(properties.command).to.equal(command); - expect(properties.has_sample_documents).to.equal(expectSampleDocs); - expect(properties.history_size).to.equal(chatContextStub.history.length); + expect(arg.properties.command).to.equal(command); + expect(arg.properties.has_sample_documents).to.equal(expectSampleDocs); + expect(arg.properties.history_size).to.equal( + chatContextStub.history.length + ); /** For docs chatbot requests, the length of the prompt would be longer as it gets the prompt history prepended.*/ if (command !== 'docs') { // Total message length includes participant as well as user prompt - expect(properties.total_message_length).to.be.greaterThan( - properties.user_input_length + expect(arg.properties.total_message_length).to.be.greaterThan( + arg.properties.user_input_length ); } // User prompt length should be at least equal to the supplied user prompt, but my occasionally // be greater - e.g. when we enhance the context. - expect(properties.user_input_length).to.be.greaterThanOrEqual( + expect(arg.properties.user_input_length).to.be.greaterThanOrEqual( chatRequest.prompt.length ); - expect(properties.internal_purpose).to.equal(expectedInternalPurpose); + expect(arg.properties.internal_purpose).to.equal(expectedInternalPurpose); }; const assertResponseTelemetry = ( @@ -183,15 +185,14 @@ suite('Participant Controller Test Suite', function () { ): void => { expect(telemetryTrackStub.callCount).to.be.greaterThan(callIndex); const call = telemetryTrackStub.getCalls()[callIndex]; - expect(call.args[0]).to.equal('Participant Response Generated'); - - const properties = call.args[1] as ParticipantResponseProperties; - - expect(properties.command).to.equal(command); - expect(properties.found_namespace).to.equal(foundNamespace); - expect(properties.has_cta).to.equal(hasCTA); - expect(properties.has_runnable_content).to.equal(hasRunnableContent); - expect(properties.output_length).to.be.greaterThan(0); + const arg = call.args[0] as ParticipantResponseGeneratedTelemetryEvent; + expect(arg.type).to.equal('Participant Response Generated'); + + expect(arg.properties.command).to.equal(command); + expect(arg.properties.found_namespace).to.equal(foundNamespace); + expect(arg.properties.has_cta).to.equal(hasCTA); + expect(arg.properties.has_runnable_content).to.equal(hasRunnableContent); + expect(arg.properties.output_length).to.be.greaterThan(0); }; beforeEach(function () { @@ -515,10 +516,9 @@ suite('Participant Controller Test Suite', function () { // Once to report welcome screen shown, second time to track the user prompt expect(telemetryTrackStub).to.have.been.calledTwice; - expect(telemetryTrackStub.firstCall.args[0]).to.equal( - TelemetryEventTypes.PARTICIPANT_WELCOME_SHOWN + expect(telemetryTrackStub.firstCall.args[0].type).to.equal( + 'Participant Welcome Shown' ); - expect(telemetryTrackStub.firstCall.args[1]).to.be.undefined; assertCommandTelemetry('query', chatRequestMock, { callIndex: 1, expectedInternalPurpose: 'namespace', @@ -551,9 +551,7 @@ suite('Participant Controller Test Suite', function () { const telemetryEvents = telemetryTrackStub .getCalls() .map((call) => call.args[0]) - .filter( - (arg) => arg === TelemetryEventTypes.PARTICIPANT_WELCOME_SHOWN - ); + .filter((arg) => arg === 'Participant Welcome Shown'); expect(telemetryEvents).to.be.empty; }); @@ -1746,13 +1744,17 @@ Schema: expect( telemetryTrackStub.getCalls() ).to.have.length.greaterThanOrEqual(2); - expect(telemetryTrackStub.firstCall.args[0]).to.equal( - TelemetryEventTypes.PARTICIPANT_RESPONSE_FAILED + + const firstTelemetryEvent = telemetryTrackStub.firstCall + .args[0] as ParticipantResponseFailedTelemetryEvent; + expect(firstTelemetryEvent.type).to.equal( + 'Participant Response Failed' ); - const properties = telemetryTrackStub.firstCall.args[1]; - expect(properties.command).to.equal('docs'); - expect(properties.error_name).to.equal('Docs Chatbot API Issue'); + expect(firstTelemetryEvent.properties.command).to.equal('docs'); + expect(firstTelemetryEvent.properties.error_name).to.equal( + 'Docs Chatbot API Issue' + ); assertResponseTelemetry('docs/copilot', { callIndex: 2, @@ -1836,17 +1838,25 @@ Schema: sendRequestStub.rejects(); const messages = sendRequestStub.firstCall.args[0]; expect(getMessageContent(messages[1])).to.equal(code.trim()); - expect(telemetryTrackStub).calledWith( - TelemetryEventTypes.EXPORT_TO_PLAYGROUND_FAILED, - { - input_length: code.trim().length, - error_name: 'streamChatResponseWithExportToLanguage', - } - ); - expect(telemetryTrackStub).not.calledWith( - TelemetryEventTypes.PARTICIPANT_RESPONSE_FAILED + const playgroundFailedTelemetryEvent = telemetryTrackStub + .getCalls() + .find((c) => c.args[0].type === 'Export To Playground Failed') + ?.args[0] as ExportToPlaygroundFailedTelemetryEvent; + expect(playgroundFailedTelemetryEvent.type).to.equal( + 'Export To Playground Failed' + ); + expect(playgroundFailedTelemetryEvent.properties.error_name).to.equal( + 'streamChatResponseWithExportToLanguage' ); + expect( + playgroundFailedTelemetryEvent.properties.input_length + ).to.equal(code.trim().length); + + const participantResponseFailedTelemetryEvent = telemetryTrackStub + .getCalls() + .find((c) => c.args[0].type === 'Participant Response Failed'); + expect(participantResponseFailedTelemetryEvent).to.be.undefined; }); test('exports selected lines of code to a playground', async function () { @@ -2268,9 +2278,9 @@ Schema: ); expect(stats.command).to.equal('generic'); - expect(stats.has_sample_documents).to.be.false; - expect(stats.user_input_length).to.equal(chatRequestMock.prompt.length); - expect(stats.total_message_length).to.equal( + expect(stats.hasSampleDocuments).to.be.false; + expect(stats.userInputLength).to.equal(chatRequestMock.prompt.length); + expect(stats.totalMessageLength).to.equal( getContentLength(messages[0]) + getContentLength(messages[1]) ); }); @@ -2335,9 +2345,9 @@ Schema: ); expect(stats.command).to.equal('query'); - expect(stats.has_sample_documents).to.be.true; - expect(stats.user_input_length).to.equal(chatRequestMock.prompt.length); - expect(stats.total_message_length).to.equal( + expect(stats.hasSampleDocuments).to.be.true; + expect(stats.userInputLength).to.equal(chatRequestMock.prompt.length); + expect(stats.totalMessageLength).to.equal( getContentLength(messages[0]) + getContentLength(messages[1]) + getContentLength(messages[2]) @@ -2345,7 +2355,7 @@ Schema: // The length of the user prompt length should be taken from the prompt supplied // by the user, even if we enhance it with sample docs and schema. - expect(stats.user_input_length).to.be.lessThan( + expect(stats.userInputLength).to.be.lessThan( getContentLength(messages[2]) ); }); @@ -2390,9 +2400,9 @@ Schema: expect(getMessageContent(messages[1])).to.include(schema); expect(stats.command).to.equal('schema'); - expect(stats.has_sample_documents).to.be.false; - expect(stats.user_input_length).to.equal(chatRequestMock.prompt.length); - expect(stats.total_message_length).to.equal( + expect(stats.hasSampleDocuments).to.be.false; + expect(stats.userInputLength).to.equal(chatRequestMock.prompt.length); + expect(stats.totalMessageLength).to.equal( getContentLength(messages[0]) + getContentLength(messages[1]) ); }); @@ -2417,9 +2427,9 @@ Schema: ); expect(stats.command).to.equal('query'); - expect(stats.has_sample_documents).to.be.false; - expect(stats.user_input_length).to.equal(chatRequestMock.prompt.length); - expect(stats.total_message_length).to.equal( + expect(stats.hasSampleDocuments).to.be.false; + expect(stats.userInputLength).to.equal(chatRequestMock.prompt.length); + expect(stats.totalMessageLength).to.equal( getContentLength(messages[0]) + getContentLength(messages[1]) ); }); @@ -2592,14 +2602,14 @@ Schema: expect(getMessageContent(messages[1])).to.contain(expectedPrompt); expect(stats.command).to.equal('query'); - expect(stats.has_sample_documents).to.be.false; - expect(stats.user_input_length).to.equal(expectedPrompt.length); - expect(stats.total_message_length).to.equal( + expect(stats.hasSampleDocuments).to.be.false; + expect(stats.userInputLength).to.equal(expectedPrompt.length); + expect(stats.totalMessageLength).to.equal( getContentLength(messages[0]) + getContentLength(messages[1]) ); // The prompt builder may add extra info, but we're only reporting the actual user input - expect(stats.user_input_length).to.be.lessThan( + expect(stats.userInputLength).to.be.lessThan( getContentLength(messages[1]) ); }); @@ -2668,17 +2678,18 @@ Schema: }); sinon.assert.calledOnce(telemetryTrackStub); - expect(telemetryTrackStub.lastCall.args[0]).to.be.equal( - 'Participant Feedback' + const telemetryEvent = telemetryTrackStub.lastCall + .args[0] as ParticipantFeedbackTelemetryEvent; + expect(telemetryEvent.type).to.be.equal('Participant Feedback'); + + expect(telemetryEvent.properties.feedback).to.be.equal('positive'); + expect(telemetryEvent.properties.reason).to.be.undefined; + expect(telemetryEvent.properties.response_type).to.be.equal( + 'askToConnect' ); - const properties = telemetryTrackStub.lastCall.args[1]; - expect(properties.feedback).to.be.equal('positive'); - expect(properties.reason).to.be.undefined; - expect(properties.response_type).to.be.equal('askToConnect'); - // Ensure we're not leaking the response content into the telemetry payload - expect(JSON.stringify(properties)) + expect(JSON.stringify(telemetryEvent.properties)) .to.not.include('creditCardNumber') .and.not.include('1234-5678-9012-3456'); }); @@ -2696,17 +2707,16 @@ Schema: } as vscode.ChatResultFeedback); sinon.assert.calledOnce(telemetryTrackStub); - expect(telemetryTrackStub.lastCall.args[0]).to.be.equal( - 'Participant Feedback' - ); + const telemetryEvent = telemetryTrackStub.lastCall + .args[0] as ParticipantFeedbackTelemetryEvent; + expect(telemetryEvent.type).to.be.equal('Participant Feedback'); - const properties = telemetryTrackStub.lastCall.args[1]; - expect(properties.feedback).to.be.equal('negative'); - expect(properties.reason).to.be.equal('incompleteCode'); - expect(properties.response_type).to.be.equal('query'); + expect(telemetryEvent.properties.feedback).to.be.equal('negative'); + expect(telemetryEvent.properties.reason).to.be.equal('incompleteCode'); + expect(telemetryEvent.properties.response_type).to.be.equal('query'); // Ensure we're not leaking the response content into the telemetry payload - expect(JSON.stringify(properties)) + expect(JSON.stringify(telemetryEvent.properties)) .to.not.include('SSN') .and.not.include('123456789'); }); @@ -2719,14 +2729,13 @@ Schema: ); sinon.assert.calledOnce(telemetryTrackStub); - expect(telemetryTrackStub.lastCall.args[0]).to.be.equal( - 'Participant Response Failed' - ); + const telemetryEvent = telemetryTrackStub.lastCall + .args[0] as ParticipantResponseFailedTelemetryEvent; + expect(telemetryEvent.type).to.be.equal('Participant Response Failed'); - const properties = telemetryTrackStub.lastCall.args[1]; - expect(properties.command).to.be.equal('query'); - expect(properties.error_code).to.be.undefined; - expect(properties.error_name).to.be.equal( + expect(telemetryEvent.properties.command).to.be.equal('query'); + expect(telemetryEvent.properties.error_code).to.be.undefined; + expect(telemetryEvent.properties.error_name).to.be.equal( 'Filtered by Responsible AI Service' ); }); @@ -2740,14 +2749,15 @@ Schema: ); sinon.assert.calledOnce(telemetryTrackStub); - expect(telemetryTrackStub.lastCall.args[0]).to.be.equal( - 'Participant Response Failed' - ); + const telemetryEvent = telemetryTrackStub.lastCall + .args[0] as ParticipantResponseFailedTelemetryEvent; + expect(telemetryEvent.type).to.be.equal('Participant Response Failed'); - const properties = telemetryTrackStub.lastCall.args[1]; - expect(properties.command).to.be.equal('docs'); - expect(properties.error_code).to.be.undefined; - expect(properties.error_name).to.be.equal('Chat Model Off Topic'); + expect(telemetryEvent.properties.command).to.be.equal('docs'); + expect(telemetryEvent.properties.error_code).to.be.undefined; + expect(telemetryEvent.properties.error_name).to.be.equal( + 'Chat Model Off Topic' + ); }); test('Reports error code when available', function () { @@ -2759,14 +2769,13 @@ Schema: ); sinon.assert.calledOnce(telemetryTrackStub); - expect(telemetryTrackStub.lastCall.args[0]).to.be.equal( - 'Participant Response Failed' - ); + const telemetryEvent = telemetryTrackStub.lastCall + .args[0] as ParticipantResponseFailedTelemetryEvent; + expect(telemetryEvent.type).to.be.equal('Participant Response Failed'); - const properties = telemetryTrackStub.lastCall.args[1]; - expect(properties.command).to.be.equal('schema'); - expect(properties.error_code).to.be.equal('NotFound'); - expect(properties.error_name).to.be.equal('Other'); + expect(telemetryEvent.properties.command).to.be.equal('schema'); + expect(telemetryEvent.properties.error_code).to.be.equal('NotFound'); + expect(telemetryEvent.properties.error_name).to.be.equal('Other'); }); }); }); diff --git a/src/test/suite/telemetry/telemetryService.test.ts b/src/test/suite/telemetry/telemetryService.test.ts index f9830ca9e..55e9fe6d7 100644 --- a/src/test/suite/telemetry/telemetryService.test.ts +++ b/src/test/suite/telemetry/telemetryService.test.ts @@ -14,7 +14,16 @@ import { DocumentSource } from '../../../documentSource'; import { mdbTestExtension } from '../stubbableMdbExtension'; import { DatabaseTreeItem, DocumentTreeItem } from '../../../explorer'; import { DataServiceStub } from '../stubs'; -import { chatResultFeedbackKindToTelemetryValue } from '../../../telemetry/telemetryService'; +import { + DocumentEditedTelemetryEvent, + DocumentUpdatedTelemetryEvent, + LinkClickedTelemetryEvent, + ParticipantFeedbackTelemetryEvent, + PlaygroundExecutedTelemetryEvent, + PlaygroundExportedToLanguageTelemetryEvent, + PlaygroundSavedTelemetryEvent, + SavedConnectionsLoadedTelemetryEvent, +} from '../../../telemetry'; // eslint-disable-next-line @typescript-eslint/no-var-requires const { version } = require('../../../../package.json'); @@ -93,7 +102,7 @@ suite('Telemetry Controller Test Suite', () => { }); test('get segment key', () => { - let segmentKey; + let segmentKey: string | undefined; try { const segmentKeyFileLocation = '../../../../constants'; @@ -187,7 +196,7 @@ suite('Telemetry Controller Test Suite', () => { test('track document saved form a tree-view event', () => { const source = DocumentSource.DOCUMENT_SOURCE_TREEVIEW; - testTelemetryService.trackDocumentUpdated(source, true); + testTelemetryService.track(new DocumentUpdatedTelemetryEvent(source, true)); sandbox.assert.calledWith( fakeSegmentAnalyticsTrack, sinon.match({ @@ -204,7 +213,7 @@ suite('Telemetry Controller Test Suite', () => { test('track document opened form playground results', () => { const source = DocumentSource.DOCUMENT_SOURCE_PLAYGROUND; - testTelemetryService.trackDocumentOpenedInEditor(source); + testTelemetryService.track(new DocumentEditedTelemetryEvent(source)); sandbox.assert.calledWith( fakeSegmentAnalyticsTrack, sinon.match({ @@ -280,7 +289,11 @@ suite('Telemetry Controller Test Suite', () => { }); test('track playground saved event', () => { - testTelemetryService.trackPlaygroundSaved('mongodbjs'); + testTelemetryService.track( + new PlaygroundSavedTelemetryEvent( + vscode.Uri.file('/users/peter/projects/test/myplayground.mongodb.js') + ) + ); sandbox.assert.calledWith( fakeSegmentAnalyticsTrack, sinon.match({ @@ -295,7 +308,9 @@ suite('Telemetry Controller Test Suite', () => { }); test('track link clicked event', () => { - testTelemetryService.trackLinkClicked('helpPanel', 'linkId'); + testTelemetryService.track( + new LinkClickedTelemetryEvent('helpPanel', 'linkId') + ); sandbox.assert.calledWith( fakeSegmentAnalyticsTrack, sinon.match({ @@ -311,11 +326,9 @@ suite('Telemetry Controller Test Suite', () => { }); test('track playground exported to language', () => { - testTelemetryService.trackPlaygroundExportedToLanguageExported({ - language: 'java', - exported_code_length: 3, - with_driver_syntax: false, - }); + testTelemetryService.track( + new PlaygroundExportedToLanguageTelemetryEvent('java', 3, false) + ); sandbox.assert.calledWith( fakeSegmentAnalyticsTrack, @@ -341,7 +354,8 @@ suite('Telemetry Controller Test Suite', () => { language: 'plaintext', }, }; - const type = testTelemetryService.getPlaygroundResultType(res); + const type = new PlaygroundExecutedTelemetryEvent(res, false, false) + .properties.type; expect(type).to.deep.equal('aggregation'); }); @@ -354,7 +368,8 @@ suite('Telemetry Controller Test Suite', () => { language: 'plaintext', }, }; - const type = testTelemetryService.getPlaygroundResultType(res); + const type = new PlaygroundExecutedTelemetryEvent(res, false, false) + .properties.type; expect(type).to.deep.equal('other'); }); @@ -367,7 +382,8 @@ suite('Telemetry Controller Test Suite', () => { language: 'plaintext', }, }; - const type = testTelemetryService.getPlaygroundResultType(res); + const type = new PlaygroundExecutedTelemetryEvent(res, false, false) + .properties.type; expect(type).to.deep.equal('other'); }); @@ -380,7 +396,8 @@ suite('Telemetry Controller Test Suite', () => { language: 'plaintext', }, }; - const type = testTelemetryService.getPlaygroundResultType(res); + const type = new PlaygroundExecutedTelemetryEvent(res, false, false) + .properties.type; expect(type).to.deep.equal('query'); }); @@ -393,7 +410,8 @@ suite('Telemetry Controller Test Suite', () => { language: 'plaintext', }, }; - const type = testTelemetryService.getPlaygroundResultType(res); + const type = new PlaygroundExecutedTelemetryEvent(res, false, false) + .properties.type; expect(type).to.deep.equal('other'); }); @@ -406,7 +424,8 @@ suite('Telemetry Controller Test Suite', () => { language: 'plaintext', }, }; - const type = testTelemetryService.getPlaygroundResultType(res); + const type = new PlaygroundExecutedTelemetryEvent(res, false, false) + .properties.type; expect(type).to.deep.equal('delete'); }); @@ -419,7 +438,8 @@ suite('Telemetry Controller Test Suite', () => { language: 'plaintext', }, }; - const type = testTelemetryService.getPlaygroundResultType(res); + const type = new PlaygroundExecutedTelemetryEvent(res, false, false) + .properties.type; expect(type).to.deep.equal('insert'); }); @@ -432,7 +452,8 @@ suite('Telemetry Controller Test Suite', () => { language: 'plaintext', }, }; - const type = testTelemetryService.getPlaygroundResultType(res); + const type = new PlaygroundExecutedTelemetryEvent(res, false, false) + .properties.type; expect(type).to.deep.equal('insert'); }); @@ -445,7 +466,8 @@ suite('Telemetry Controller Test Suite', () => { language: 'plaintext', }, }; - const type = testTelemetryService.getPlaygroundResultType(res); + const type = new PlaygroundExecutedTelemetryEvent(res, false, false) + .properties.type; expect(type).to.deep.equal('other'); }); @@ -458,7 +480,8 @@ suite('Telemetry Controller Test Suite', () => { language: 'plaintext', }, }; - const type = testTelemetryService.getPlaygroundResultType(res); + const type = new PlaygroundExecutedTelemetryEvent(res, false, false) + .properties.type; expect(type).to.deep.equal('other'); }); @@ -471,7 +494,8 @@ suite('Telemetry Controller Test Suite', () => { language: 'plaintext', }, }; - const type = testTelemetryService.getPlaygroundResultType(res); + const type = new PlaygroundExecutedTelemetryEvent(res, false, false) + .properties.type; expect(type).to.deep.equal('other'); }); @@ -484,7 +508,8 @@ suite('Telemetry Controller Test Suite', () => { language: 'plaintext', }, }; - const type = testTelemetryService.getPlaygroundResultType(res); + const type = new PlaygroundExecutedTelemetryEvent(res, false, false) + .properties.type; expect(type).to.deep.equal('update'); }); @@ -497,7 +522,8 @@ suite('Telemetry Controller Test Suite', () => { language: 'plaintext', }, }; - const type = testTelemetryService.getPlaygroundResultType(res); + const type = new PlaygroundExecutedTelemetryEvent(res, false, false) + .properties.type; expect(type).to.deep.equal('other'); }); }); @@ -669,13 +695,15 @@ suite('Telemetry Controller Test Suite', () => { }); test.skip('track saved connections loaded', () => { - testTelemetryService.trackSavedConnectionsLoaded({ - saved_connections: 3, - loaded_connections: 3, - preset_connections: 3, - connections_with_secrets_in_keytar: 0, - connections_with_secrets_in_secret_storage: 3, - }); + testTelemetryService.track( + new SavedConnectionsLoadedTelemetryEvent({ + savedConnections: 3, + loadedConnections: 3, + presetConnections: 3, + connectionsWithSecretsInKeytar: 0, + connectionsWithSecretsInSecretStorage: 3, + }) + ); sandbox.assert.calledWith( fakeSegmentAnalyticsTrack, @@ -693,27 +721,6 @@ suite('Telemetry Controller Test Suite', () => { ); }); - test('track failed keytar secrets migrations', () => { - testTelemetryService.trackKeytarSecretsMigrationFailed({ - saved_connections: 3, - loaded_connections: 3, - connections_with_failed_keytar_migration: 1, - }); - - sandbox.assert.calledWith( - fakeSegmentAnalyticsTrack, - sinon.match({ - anonymousId, - event: 'Keytar Secrets Migration Failed', - properties: { - saved_connections: 3, - loaded_connections: 3, - connections_with_failed_keytar_migration: 1, - }, - }) - ); - }); - function enumKeys< TEnum extends object, TKey extends keyof TEnum = keyof TEnum @@ -724,9 +731,10 @@ suite('Telemetry Controller Test Suite', () => { test('ChatResultFeedbackKind to TelemetryFeedbackKind maps all values', () => { for (const kind of enumKeys(vscode.ChatResultFeedbackKind)) { expect( - chatResultFeedbackKindToTelemetryValue( - vscode.ChatResultFeedbackKind[kind] - ), + new ParticipantFeedbackTelemetryEvent( + vscode.ChatResultFeedbackKind[kind], + 'generic' + ).properties.feedback, `Expect ${kind} to produce a concrete telemetry value` ).to.not.be.undefined; } diff --git a/src/test/suite/views/webviewController.test.ts b/src/test/suite/views/webviewController.test.ts index f0cb964dd..a88ace2bd 100644 --- a/src/test/suite/views/webviewController.test.ts +++ b/src/test/suite/views/webviewController.test.ts @@ -10,7 +10,7 @@ import { mdbTestExtension } from '../stubbableMdbExtension'; import { MESSAGE_TYPES } from '../../../views/webview-app/extension-app-message-constants'; import { StatusView } from '../../../views'; import { StorageController } from '../../../storage'; -import TelemetryService from '../../../telemetry/telemetryService'; +import TelemetryService from '../../../telemetry'; import { ExtensionContextStub } from '../stubs'; import { TEST_DATABASE_URI } from '../dbTestHelper'; import WebviewController, { diff --git a/src/utils/playground.ts b/src/utils/playground.ts index 392fbf47f..e568b0800 100644 --- a/src/utils/playground.ts +++ b/src/utils/playground.ts @@ -103,20 +103,6 @@ export const getAllText = (): string => { return vscode.window.activeTextEditor?.document.getText().trim() || ''; }; -export const getPlaygroundExtensionForTelemetry = ( - fileUri?: vscode.Uri -): string => { - let fileType = 'other'; - - if (fileUri?.fsPath.match(/\.(mongodb\.js)$/gi)) { - fileType = 'mongodbjs'; - } else if (fileUri?.fsPath.match(/\.(mongodb)$/gi)) { - fileType = 'mongodb'; - } - - return fileType; -}; - export const getPlaygrounds = async ({ fsPath, excludeFromPlaygroundsSearch, diff --git a/src/views/webviewController.ts b/src/views/webviewController.ts index fbf0c81cb..02c21080c 100644 --- a/src/views/webviewController.ts +++ b/src/views/webviewController.ts @@ -15,9 +15,13 @@ import { } from './webview-app/extension-app-message-constants'; import { openLink } from '../utils/linkHelper'; import type { StorageController } from '../storage'; -import type TelemetryService from '../telemetry/telemetryService'; +import type TelemetryService from '../telemetry'; import { getFeatureFlagsScript } from '../featureFlags'; -import { TelemetryEventTypes } from '../telemetry/telemetryService'; +import { + ConnectionEditedTelemetryEvent, + LinkClickedTelemetryEvent, + OpenEditConnectionTelemetryEvent, +} from '../telemetry'; import type { FileChooserOptions } from './webview-app/use-connection-form'; const log = createLogger('webview controller'); @@ -248,7 +252,7 @@ export default class WebviewController { connection: message.connectionInfo, isEditingConnection: true, }); - this._telemetryService.track(TelemetryEventTypes.CONNECTION_EDITED); + this._telemetryService.track(new ConnectionEditedTelemetryEvent()); return; case MESSAGE_TYPES.OPEN_FILE_CHOOSER: await this.handleWebviewOpenFileChooserAttempt({ @@ -292,7 +296,9 @@ export default class WebviewController { } return; case MESSAGE_TYPES.EXTENSION_LINK_CLICKED: - this._telemetryService.trackLinkClicked(message.screen, message.linkId); + this._telemetryService.track( + new LinkClickedTelemetryEvent(message.screen, message.linkId) + ); return; case MESSAGE_TYPES.RENAME_ACTIVE_CONNECTION: if (this._connectionController.isCurrentlyConnected()) { @@ -360,7 +366,7 @@ export default class WebviewController { // Wait for the panel to open. await new Promise((resolve) => setTimeout(resolve, 200)); - this._telemetryService.track(TelemetryEventTypes.OPEN_EDIT_CONNECTION); + this._telemetryService.track(new OpenEditConnectionTelemetryEvent()); void webviewPanel.webview.postMessage({ command: MESSAGE_TYPES.OPEN_EDIT_CONNECTION, From 351ef36764b247930810b0aa2d459216ced72bda Mon Sep 17 00:00:00 2001 From: Nikola Irinchev Date: Wed, 22 Jan 2025 18:04:54 +0100 Subject: [PATCH 2/3] fix lint --- src/connectionController.ts | 1 - src/test/suite/editors/collectionDocumentsProvider.test.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/src/connectionController.ts b/src/connectionController.ts index cbb3c8149..090a2276a 100644 --- a/src/connectionController.ts +++ b/src/connectionController.ts @@ -28,7 +28,6 @@ import type { import { ConnectionStorage } from './storage/connectionStorage'; import LINKS from './utils/links'; import { isAtlasStream } from 'mongodb-build-info'; -import { DocumentSource } from './documentSource'; import type { ConnectionTreeItem } from './explorer'; import { PresetConnectionEditedTelemetryEvent } from './telemetry'; diff --git a/src/test/suite/editors/collectionDocumentsProvider.test.ts b/src/test/suite/editors/collectionDocumentsProvider.test.ts index f23aadd74..4650c9c2f 100644 --- a/src/test/suite/editors/collectionDocumentsProvider.test.ts +++ b/src/test/suite/editors/collectionDocumentsProvider.test.ts @@ -20,7 +20,6 @@ import { import TelemetryService from '../../../telemetry'; import { TEST_DATABASE_URI } from '../dbTestHelper'; import { ExtensionContextStub, mockTextEditor } from '../stubs'; -import { expect } from 'chai'; const mockDocumentsAsJsonString = `[ { From 0efec2a5bce74a85b12d7eb0f543d1d76e46579b Mon Sep 17 00:00:00 2001 From: Nikola Irinchev Date: Thu, 23 Jan 2025 12:38:51 +0100 Subject: [PATCH 3/3] fix CR comments --- src/connectionController.ts | 2 +- src/editors/editorsController.ts | 2 +- src/editors/mongoDBDocumentService.ts | 2 +- src/editors/playgroundController.ts | 2 +- src/explorer/helpExplorer.ts | 2 +- src/explorer/helpTree.ts | 2 +- src/mdbExtensionController.ts | 2 +- src/participant/participant.ts | 2 +- src/telemetry/index.ts | 4 +- src/telemetry/telemetryEvents.ts | 270 +++++++++--------- src/telemetry/telemetryService.ts | 2 +- src/test/suite/connectionController.test.ts | 2 +- .../activeConnectionCodeLensProvider.test.ts | 2 +- .../collectionDocumentsProvider.test.ts | 2 +- .../editDocumentCodeLensProvider.test.ts | 2 +- .../exportToLanguageCodeLensProvider.test.ts | 2 +- .../editors/mongoDBDocumentService.test.ts | 2 +- .../editors/playgroundController.test.ts | 2 +- .../editors/playgroundResultProvider.test.ts | 2 +- .../language/languageServerController.test.ts | 2 +- src/test/suite/oidc.test.ts | 2 +- .../suite/participant/participant.test.ts | 2 +- .../suite/views/webviewController.test.ts | 2 +- src/views/webviewController.ts | 2 +- 24 files changed, 151 insertions(+), 167 deletions(-) diff --git a/src/connectionController.ts b/src/connectionController.ts index 090a2276a..f7f396326 100644 --- a/src/connectionController.ts +++ b/src/connectionController.ts @@ -19,7 +19,7 @@ import { createLogger } from './logging'; import formatError from './utils/formatError'; import type { StorageController } from './storage'; import type { StatusView } from './views'; -import type TelemetryService from './telemetry'; +import type { TelemetryService } from './telemetry'; import { openLink } from './utils/linkHelper'; import type { ConnectionSource, diff --git a/src/editors/editorsController.ts b/src/editors/editorsController.ts index 74d8f8008..67285595e 100644 --- a/src/editors/editorsController.ts +++ b/src/editors/editorsController.ts @@ -31,7 +31,7 @@ import type PlaygroundController from './playgroundController'; import type PlaygroundResultProvider from './playgroundResultProvider'; import { PLAYGROUND_RESULT_SCHEME } from './playgroundResultProvider'; import { StatusView } from '../views'; -import type TelemetryService from '../telemetry'; +import type { TelemetryService } from '../telemetry'; import type { QueryWithCopilotCodeLensProvider } from './queryWithCopilotCodeLensProvider'; const log = createLogger('editors controller'); diff --git a/src/editors/mongoDBDocumentService.ts b/src/editors/mongoDBDocumentService.ts index e1d81d2de..a67a9b446 100644 --- a/src/editors/mongoDBDocumentService.ts +++ b/src/editors/mongoDBDocumentService.ts @@ -7,7 +7,7 @@ import { DocumentSource } from '../documentSource'; import type { EditDocumentInfo } from '../types/editDocumentInfoType'; import formatError from '../utils/formatError'; import type { StatusView } from '../views'; -import type TelemetryService from '../telemetry'; +import type { TelemetryService } from '../telemetry'; import { getEJSON } from '../utils/ejson'; import { DocumentUpdatedTelemetryEvent } from '../telemetry'; diff --git a/src/editors/playgroundController.ts b/src/editors/playgroundController.ts index f89dbbe53..4268c82b2 100644 --- a/src/editors/playgroundController.ts +++ b/src/editors/playgroundController.ts @@ -34,7 +34,7 @@ import { import playgroundSearchTemplate from '../templates/playgroundSearchTemplate'; import playgroundTemplate from '../templates/playgroundTemplate'; import type { StatusView } from '../views'; -import type TelemetryService from '../telemetry'; +import type { TelemetryService } from '../telemetry'; import { isPlayground, getSelectedText, getAllText } from '../utils/playground'; import type ExportToLanguageCodeLensProvider from './exportToLanguageCodeLensProvider'; import { playgroundFromDatabaseTreeItemTemplate } from '../templates/playgroundFromDatabaseTreeItemTemplate'; diff --git a/src/explorer/helpExplorer.ts b/src/explorer/helpExplorer.ts index 1dbf3b374..2e6cdf2fb 100644 --- a/src/explorer/helpExplorer.ts +++ b/src/explorer/helpExplorer.ts @@ -1,6 +1,6 @@ import * as vscode from 'vscode'; import HelpTree from './helpTree'; -import type TelemetryService from '../telemetry'; +import type { TelemetryService } from '../telemetry'; export default class HelpExplorer { _treeController: HelpTree; diff --git a/src/explorer/helpTree.ts b/src/explorer/helpTree.ts index dc64d981f..ca0f5bc94 100644 --- a/src/explorer/helpTree.ts +++ b/src/explorer/helpTree.ts @@ -1,7 +1,7 @@ import * as vscode from 'vscode'; import path from 'path'; import { getImagesPath } from '../extensionConstants'; -import type TelemetryService from '../telemetry'; +import type { TelemetryService } from '../telemetry'; import { openLink } from '../utils/linkHelper'; import LINKS from '../utils/links'; import { LinkClickedTelemetryEvent } from '../telemetry'; diff --git a/src/mdbExtensionController.ts b/src/mdbExtensionController.ts index 1432d01f7..dad644070 100644 --- a/src/mdbExtensionController.ts +++ b/src/mdbExtensionController.ts @@ -33,7 +33,7 @@ import launchMongoShell from './commands/launchMongoShell'; import type SchemaTreeItem from './explorer/schemaTreeItem'; import { StatusView } from './views'; import { StorageController, StorageVariables } from './storage'; -import TelemetryService from './telemetry'; +import { TelemetryService } from './telemetry'; import type PlaygroundsTreeItem from './explorer/playgroundsTreeItem'; import PlaygroundResultProvider from './editors/playgroundResultProvider'; import WebviewController from './views/webviewController'; diff --git a/src/participant/participant.ts b/src/participant/participant.ts index 9b57cfaa8..fd36fa7d5 100644 --- a/src/participant/participant.ts +++ b/src/participant/participant.ts @@ -46,7 +46,7 @@ import { PlaygroundExportedToLanguageTelemetryEvent, } from '../telemetry'; import { DocsChatbotAIService } from './docsChatbotAIService'; -import type TelemetryService from '../telemetry'; +import type { TelemetryService } from '../telemetry'; import formatError from '../utils/formatError'; import { getContent, type ModelInput } from './prompts/promptBase'; import { processStreamWithIdentifiers } from './streamParsing'; diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts index a9495a57d..b06c36a30 100644 --- a/src/telemetry/index.ts +++ b/src/telemetry/index.ts @@ -1,4 +1,2 @@ -import TelemetryService from './telemetryService'; export * from './telemetryEvents'; - -export default TelemetryService; +export { TelemetryService } from './telemetryService'; diff --git a/src/telemetry/telemetryEvents.ts b/src/telemetry/telemetryEvents.ts index e020c94e6..4c4554fea 100644 --- a/src/telemetry/telemetryEvents.ts +++ b/src/telemetry/telemetryEvents.ts @@ -18,13 +18,16 @@ type PlaygroundFileType = 'other' | 'mongodbjs' | 'mongodb'; type TelemetryFeedbackKind = 'positive' | 'negative' | undefined; +/** + * The purpose of the internal prompt - e.g. 'intent', 'namespace' + */ export type InternalPromptPurpose = 'intent' | 'namespace' | undefined; export type ParticipantTelemetryMetadata = { - // The source of the participant prompt - e.g. 'codelens', 'treeview', etc. + /** The source of the participant prompt - e.g. 'codelens', 'treeview', etc. */ source: DocumentSource; - // Additional details about the source - e.g. if it's 'treeview', the detail can be 'database' or 'collection'. + /** Additional details about the source - e.g. if it's 'treeview', the detail can be 'database' or 'collection'. */ source_details: DocumentSourceDetails; }; @@ -51,7 +54,7 @@ function getPlaygroundFileTypeFromUri( return fileType; } -type PlaygrdoundType = +type PlaygroundType = | 'search' | 'createCollection' | 'createDatabase' @@ -64,52 +67,22 @@ type PlaygrdoundType = | 'fromCollectionTreeItem' | 'crud'; -type TelemetryEventType = - | 'Playground Code Executed' - | 'Link Clicked' - | 'Command Run' - | 'New Connection' - | 'Connection Edited' - | 'Open Edit Connection' - | 'Playground Saved' - | 'Playground Loaded' - | 'Document Updated' - | 'Document Edited' - | 'Playground Exported To Language' - | 'Playground Created' - | 'Export To Playground Failed' - | 'Saved Connections Loaded' - | 'Participant Feedback' - | 'Participant Welcome Shown' - | 'Participant Response Failed' - /** Tracks all submitted prompts */ - | 'Participant Prompt Submitted' - /** Tracks prompts that were submitted as a result of an action other than - * the user typing the message, such as clicking on an item in tree view or a codelens */ - | 'Participant Prompt Submitted From Action' - /** Tracks when a new chat was opened from an action such as clicking on a tree view. */ - | 'Participant Chat Opened From Action' - /** Tracks after a participant interacts with the input box we open to let the user write the prompt for participant. */ - | 'Participant Inbox Box Submitted' - | 'Participant Response Generated' - | 'Preset Connection Edited'; - abstract class TelemetryEventBase { - abstract type: TelemetryEventType; + abstract type: string; abstract properties: Record; } -// Reported when a playground file is run +/** Reported when a playground file is run */ export class PlaygroundExecutedTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Playground Code Executed'; + type = 'Playground Code Executed'; properties: { - // The type of the executed operation, e.g. 'insert', 'update', 'delete', 'query', 'aggregation', 'other' + /** The type of the executed operation, e.g. 'insert', 'update', 'delete', 'query', 'aggregation', 'other' */ type: string | null; - // Whether the entire script was run or just a part of it + /** Whether the entire script was run or just a part of it */ partial: boolean; - // Whether an error occurred during execution + /** Whether an error occurred during execution */ error: boolean; }; @@ -149,14 +122,14 @@ export class PlaygroundExecutedTelemetryEvent implements TelemetryEventBase { } } -// Reported when a user clicks a hyperlink - e.g. from the Help pane +/** Reported when a user clicks a hyperlink - e.g. from the Help pane */ export class LinkClickedTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Link Clicked'; + type = 'Link Clicked'; properties: { - // The screen where the link was clicked + /** The screen where the link was clicked */ screen: string; - // The ID of the clicked link - e.g. `whatsNew`, `extensionDocumentation`, etc. + /** The ID of the clicked link - e.g. `whatsNew`, `extensionDocumentation`, etc. */ link_id: string; }; @@ -165,14 +138,16 @@ export class LinkClickedTelemetryEvent implements TelemetryEventBase { } } -// Reported when any command is run by the user. Commands are the building blocks -// of the extension and can be executed either by clicking a UI element or by opening -// the command pallette (CMD+Shift+P). This event is likely to duplicate other events -// as it's fired automatically, regardless of other more-specific events. +/** + * Reported when any command is run by the user. Commands are the building blocks + * of the extension and can be executed either by clicking a UI element or by opening + * the command pallette (CMD+Shift+P). This event is likely to duplicate other events + * as it's fired automatically, regardless of other more-specific events. + */ export class CommandRunTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Command Run'; + type = 'Command Run'; properties: { - // The command that was executed - e.g. `mdb.connect`, `mdb.openMongoDBIssueReporter`, etc. + /** The command that was executed - e.g. `mdb.connect`, `mdb.openMongoDBIssueReporter`, etc. */ command: ExtensionCommand; }; @@ -181,9 +156,9 @@ export class CommandRunTelemetryEvent implements TelemetryEventBase { } } -// Reported every time we connect to a cluster/db +/** Reported every time we connect to a cluster/db */ export class NewConnectionTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'New Connection'; + type = 'New Connection'; properties: NewConnectionTelemetryEventProperties; constructor(properties: NewConnectionTelemetryEventProperties) { @@ -191,23 +166,23 @@ export class NewConnectionTelemetryEvent implements TelemetryEventBase { } } -// Reported when a connection is edited +/** Reported when a connection is edited */ export class ConnectionEditedTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Connection Edited'; + type = 'Connection Edited'; properties = {}; } -// Reported when the user opens the connection editor +/** Reported when the user opens the connection editor */ export class OpenEditConnectionTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Open Edit Connection'; + type = 'Open Edit Connection'; properties = {}; } -// Reported when a playground file is saved +/** Reported when a playground file is saved */ export class PlaygroundSavedTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Playground Saved'; + type = 'Playground Saved'; properties: { - // The type of the file, e.g. 'mongodbjs' for .mongodb.js or 'mongodb' for .mongodb + /** The type of the file, e.g. 'mongodbjs' for .mongodb.js or 'mongodb' for .mongodb */ file_type: PlaygroundFileType; }; @@ -216,11 +191,11 @@ export class PlaygroundSavedTelemetryEvent implements TelemetryEventBase { } } -// Reported when a playground file is opened +/** Reported when a playground file is opened */ export class PlaygroundLoadedTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Playground Loaded'; + type = 'Playground Loaded'; properties: { - // The type of the file, e.g. 'mongodbjs' for .mongodb.js or 'mongodb' for .mongodb + /** The type of the file, e.g. 'mongodbjs' for .mongodb.js or 'mongodb' for .mongodb */ file_type: PlaygroundFileType; }; @@ -229,14 +204,14 @@ export class PlaygroundLoadedTelemetryEvent implements TelemetryEventBase { } } -// Reported when a document is saved (e.g. when the user edits a document from a collection) +/** Reported when a document is saved (e.g. when the user edits a document from a collection) */ export class DocumentUpdatedTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Document Updated'; + type = 'Document Updated'; properties: { - // The source of the document update, e.g. 'editor', 'tree_view', etc. + /** The source of the document update, e.g. 'editor', 'tree_view', etc. */ source: DocumentSource; - // Whether the operation was successful + /** Whether the operation was successful */ success: boolean; }; @@ -245,11 +220,11 @@ export class DocumentUpdatedTelemetryEvent implements TelemetryEventBase { } } -// Reported when a document is opened in the editor, e.g. from a query results view +/** Reported when a document is opened in the editor, e.g. from a query results view */ export class DocumentEditedTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Document Edited'; + type = 'Document Edited'; properties: { - // The source of the document - e.g. codelens, treeview, etc. + /** The source of the document - e.g. codelens, treeview, etc. */ source: DocumentSource; }; @@ -258,19 +233,19 @@ export class DocumentEditedTelemetryEvent implements TelemetryEventBase { } } -// Reported when a playground file is exported to a language +/** Reported when a playground file is exported to a language */ export class PlaygroundExportedToLanguageTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Playground Exported To Language'; + type = 'Playground Exported To Language'; properties: { - // The target language of the export + /** The target language of the export */ language: string; - // The length of the exported code + /** The length of the exported code */ exported_code_length: number; - // Whether the user opted to include driver syntax (e.g. import statements) + /** Whether the user opted to include driver syntax (e.g. import statements) */ with_driver_syntax: boolean; }; @@ -287,29 +262,31 @@ export class PlaygroundExportedToLanguageTelemetryEvent } } -// Reported when a new playground is created +/** Reported when a new playground is created */ export class PlaygroundCreatedTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Playground Created'; + type = 'Playground Created'; properties: { - // The playground type - e.g. 'search', 'createCollection', 'createDatabase', etc. This is typically - // indicative of the element the user clicked to create the playground. - playground_type: PlaygrdoundType; + /** + * The playground type - e.g. 'search', 'createCollection', 'createDatabase', etc. This is typically + * indicative of the element the user clicked to create the playground. + */ + playground_type: PlaygroundType; }; - constructor(playgroundType: PlaygrdoundType) { + constructor(playgroundType: PlaygroundType) { this.properties = { playground_type: playgroundType }; } } -// Reported when exporting to playground fails +/** Reported when exporting to playground fails */ export class ExportToPlaygroundFailedTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Export To Playground Failed'; + type = 'Export To Playground Failed'; properties: { - // The length of the playground code + /** The length of the playground code */ input_length: number | undefined; - // The name of the error that occurred + /** The name of the error that occurred */ error_name?: ExportToPlaygroundError; }; @@ -321,29 +298,33 @@ export class ExportToPlaygroundFailedTelemetryEvent } } -// Reported when saved connections are loaded from disk. This is currently disabled -// due to the large volume of events. +/** + * Reported when saved connections are loaded from disk. This is currently disabled + * due to the large volume of events. + */ export class SavedConnectionsLoadedTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Saved Connections Loaded'; + type = 'Saved Connections Loaded'; properties: { - // Total number of connections saved on disk + /** Total number of connections saved on disk */ saved_connections: number; - // Total number of connections from preset settings + /** Total number of connections from preset settings */ preset_connections: number; - // Total number of connections that extension was able to load, it might - // differ from saved_connections since there might be failures in loading - // secrets for a connection in which case we don't list the connections in the - // list of loaded connections. + /** + * Total number of connections that extension was able to load, it might + * differ from saved_connections since there might be failures in loading + * secrets for a connection in which case we don't list the connections in the + * list of loaded connections. + * */ loaded_connections: number; - // Total number of connections that have secrets stored in keytar + /** Total number of connections that have secrets stored in keytar */ connections_with_secrets_in_keytar: number; - // Total number of connections that have secrets stored in secret storage + /** Total number of connections that have secrets stored in secret storage */ connections_with_secrets_in_secret_storage: number; }; @@ -371,18 +352,19 @@ export class SavedConnectionsLoadedTelemetryEvent } } -// Reported when the user provides feedback to the chatbot on a response +/** Reported when the user provides feedback to the chatbot on a response */ export class ParticipantFeedbackTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Participant Feedback'; + type = 'Participant Feedback'; properties: { - // The type of feedback provided - e.g. 'positive', 'negative' + /** The type of feedback provided - e.g. 'positive', 'negative' */ feedback: TelemetryFeedbackKind; - // The response type that the feedback was provided for - e.g. 'query', 'schema', 'docs' + /** The response type that the feedback was provided for - e.g. 'query', 'schema', 'docs' */ response_type: ParticipantResponseType; - // If the feedback was negative, the reason for the negative feedback. It's picked from - // a set of predefined options and not a free-form text field. + /** If the feedback was negative, the reason for the negative feedback. It's picked from + * a set of predefined options and not a free-form text field. + * */ reason?: String; }; @@ -412,30 +394,30 @@ export class ParticipantFeedbackTelemetryEvent implements TelemetryEventBase { } } -// Reported when the participant welcome message is shown +/** Reported when the participant welcome message is shown */ export class ParticipantWelcomeShownTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Participant Welcome Shown'; + type = 'Participant Welcome Shown'; properties = {}; } -// Reported when a participant response fails +/** Reported when a participant response fails */ export class ParticipantResponseFailedTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Participant Response Failed'; + type = 'Participant Response Failed'; properties: { - // The type of the command that failed - e.g. 'query', 'schema', 'docs' + /** The type of the command that failed - e.g. 'query', 'schema', 'docs' */ command: ParticipantResponseType; - // The error code that caused the failure + /** The error code that caused the failure */ error_code?: string; - // The name of the error that caused the failure + /** The name of the error that caused the failure */ error_name: ParticipantErrorTypes; - // Additional details about the error if any. + /** Additional details about the error if any. */ error_details?: string; }; @@ -454,29 +436,30 @@ export class ParticipantResponseFailedTelemetryEvent } } -// Reported when a participant prompt is submitted +/** Reported when a participant prompt is submitted */ export class ParticipantPromptSubmittedTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Participant Prompt Submitted'; + type = 'Participant Prompt Submitted'; properties: { - // The type of the command that was submitted - e.g. 'query', 'schema', 'docs' + /** The type of the command that was submitted - e.g. 'query', 'schema', 'docs' */ command: ParticipantCommandType; - // The length of the user input + /** The length of the user input */ user_input_length: number; - // The total length of the message - i.e. user input + participant prompt + /** The total length of the message - i.e. user input + participant prompt */ total_message_length: number; - // Whether the prompt has sample documents + /** Whether the prompt has sample documents */ has_sample_documents: boolean; - // The size of the history + /** The size of the history */ history_size: number; - // For internal prompts - e.g. trying to extract the 'intent', 'namespace' or the - // namespace from the chat history. + /** For internal prompts - e.g. trying to extract the 'intent', 'namespace' or the + * namespace from the chat history. + */ internal_purpose: InternalPromptPurpose; }; @@ -499,17 +482,19 @@ export class ParticipantPromptSubmittedTelemetryEvent } } -// Reported when a participant prompt is submitted from an action other than typing directly. -// This is typically one of the activation points - e.g. clicking on the tree view, a codelens, etc. +/** + * Reported when a participant prompt is submitted from an action other than typing directly. + * This is typically one of the activation points - e.g. clicking on the tree view, a codelens, etc. + */ export class ParticipantPromptSubmittedFromActionTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Participant Prompt Submitted From Action'; + type = 'Participant Prompt Submitted From Action'; properties: ParticipantTelemetryMetadata & { - // The length of the input + /** The length of the input */ input_length: number; - // The command we're requesting - e.g. 'query', 'schema', 'docs' + /** The command we're requesting - e.g. 'query', 'schema', 'docs' */ command: ParticipantRequestType; }; @@ -526,13 +511,13 @@ export class ParticipantPromptSubmittedFromActionTelemetryEvent } } -// Reported when a new chat is initiated from an activation point in the extension (e.g. the database tree view) +/** Reported when a new chat is initiated from an activation point in the extension (e.g. the database tree view) */ export class ParticipantChatOpenedFromActionTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Participant Chat Opened From Action'; + type = 'Participant Chat Opened From Action'; properties: ParticipantTelemetryMetadata & { - // The command - if any - we're opening a chat for - e.g. 'query', 'schema', 'docs' + /** The command - if any - we're opening a chat for - e.g. 'query', 'schema', 'docs' */ command?: ParticipantCommandType; }; @@ -544,19 +529,19 @@ export class ParticipantChatOpenedFromActionTelemetryEvent } } -// Reported when we open an input box to ask the user for a message that we'll send to copilot +/** Reported when we open an input box to ask the user for a message that we'll send to copilot */ export class ParticipantInputBoxSubmittedTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Participant Inbox Box Submitted'; + type = 'Participant Inbox Box Submitted'; properties: ParticipantTelemetryMetadata & { - // The supplied input length + /** The supplied input length */ input_length: number; - // Whether the input was dismissed + /** Whether the input was dismissed */ dismissed: boolean; - // The command we're requesting - e.g. 'query', 'schema', 'docs' + /** The command we're requesting - e.g. 'query', 'schema', 'docs' */ command?: ParticipantCommandType; }; @@ -574,25 +559,25 @@ export class ParticipantInputBoxSubmittedTelemetryEvent } } -// Reported when a participant response is generated +/** Reported when a participant response is generated */ export class ParticipantResponseGeneratedTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Participant Response Generated'; + type = 'Participant Response Generated'; properties: { - // The type of the command that was requested - e.g. 'query', 'schema', 'docs' + /** The type of the command that was requested - e.g. 'query', 'schema', 'docs' */ command: ParticipantResponseType; - // Whether the response has a call to action (e.g. 'Open in playground' button) + /** Whether the response has a call to action (e.g. 'Open in playground' button) */ has_cta: boolean; - // Whether the response has runnable content (e.g. a code block) + /** Whether the response has runnable content (e.g. a code block) */ has_runnable_content: boolean; - // Whether the response contains namespace information + /** Whether the response contains namespace information */ found_namespace: boolean; - // The length of the output + /** The length of the output */ output_length: number; }; @@ -619,17 +604,18 @@ export class ParticipantResponseGeneratedTelemetryEvent } } -// Reported when a preset connection is edited +/** Reported when a preset connection is edited */ export class PresetConnectionEditedTelemetryEvent implements TelemetryEventBase { - type: TelemetryEventType = 'Preset Connection Edited'; + type = 'Preset Connection Edited'; properties: { - // The source of the interaction - currently, only treeview + /** The source of the interaction - currently, only treeview */ source: Extract; - // Additional details about the source - e.g. if it's a specific connection element, - // it'll be 'tree_item', otherwise it'll be 'header'. + /** Additional details about the source - e.g. if it's a specific connection element, + * it'll be 'tree_item', otherwise it'll be 'header'. + */ source_details: 'tree_item' | 'header'; }; diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts index 5ce237ce3..fd6d0b730 100644 --- a/src/telemetry/telemetryService.ts +++ b/src/telemetry/telemetryService.ts @@ -30,7 +30,7 @@ export type SegmentProperties = { /** * This controller manages telemetry. */ -export default class TelemetryService { +export class TelemetryService { _segmentAnalytics?: SegmentAnalytics; _segmentAnonymousId: string; _segmentKey?: string; // The segment API write key. diff --git a/src/test/suite/connectionController.test.ts b/src/test/suite/connectionController.test.ts index 943a0950f..92ad2a1ac 100644 --- a/src/test/suite/connectionController.test.ts +++ b/src/test/suite/connectionController.test.ts @@ -19,7 +19,7 @@ import { SecretStorageLocation, } from '../../storage/storageController'; import { StatusView } from '../../views'; -import TelemetryService from '../../telemetry'; +import { TelemetryService } from '../../telemetry'; import { ExtensionContextStub } from './stubs'; import { TEST_DATABASE_URI, diff --git a/src/test/suite/editors/activeConnectionCodeLensProvider.test.ts b/src/test/suite/editors/activeConnectionCodeLensProvider.test.ts index 62f7a5708..58dc93f2c 100644 --- a/src/test/suite/editors/activeConnectionCodeLensProvider.test.ts +++ b/src/test/suite/editors/activeConnectionCodeLensProvider.test.ts @@ -10,7 +10,7 @@ import ConnectionController from '../../../connectionController'; import { StatusView } from '../../../views'; import { StorageController } from '../../../storage'; import { ExtensionContextStub } from '../stubs'; -import TelemetryService from '../../../telemetry'; +import { TelemetryService } from '../../../telemetry'; import { TEST_DATABASE_URI } from '../dbTestHelper'; suite('Active Connection CodeLens Provider Test Suite', () => { diff --git a/src/test/suite/editors/collectionDocumentsProvider.test.ts b/src/test/suite/editors/collectionDocumentsProvider.test.ts index 4650c9c2f..95b78a05e 100644 --- a/src/test/suite/editors/collectionDocumentsProvider.test.ts +++ b/src/test/suite/editors/collectionDocumentsProvider.test.ts @@ -17,7 +17,7 @@ import { SecretStorageLocation, StorageLocation, } from '../../../storage/storageController'; -import TelemetryService from '../../../telemetry'; +import { TelemetryService } from '../../../telemetry'; import { TEST_DATABASE_URI } from '../dbTestHelper'; import { ExtensionContextStub, mockTextEditor } from '../stubs'; diff --git a/src/test/suite/editors/editDocumentCodeLensProvider.test.ts b/src/test/suite/editors/editDocumentCodeLensProvider.test.ts index b4f150303..3643aa61e 100644 --- a/src/test/suite/editors/editDocumentCodeLensProvider.test.ts +++ b/src/test/suite/editors/editDocumentCodeLensProvider.test.ts @@ -11,7 +11,7 @@ import EditDocumentCodeLensProvider from '../../../editors/editDocumentCodeLensP import { mockTextEditor } from '../stubs'; import { StatusView } from '../../../views'; import { StorageController } from '../../../storage'; -import TelemetryService from '../../../telemetry'; +import { TelemetryService } from '../../../telemetry'; import { ExtensionContextStub } from '../stubs'; suite('Edit Document Code Lens Provider Test Suite', () => { diff --git a/src/test/suite/editors/exportToLanguageCodeLensProvider.test.ts b/src/test/suite/editors/exportToLanguageCodeLensProvider.test.ts index 268431f75..25b7f5662 100644 --- a/src/test/suite/editors/exportToLanguageCodeLensProvider.test.ts +++ b/src/test/suite/editors/exportToLanguageCodeLensProvider.test.ts @@ -7,7 +7,7 @@ import ExportToLanguageCodeLensProvider, { import PlaygroundResultProvider from '../../../editors/playgroundResultProvider'; import StorageController from '../../../storage/storageController'; import { ExtensionContextStub } from '../stubs'; -import TelemetryService from '../../../telemetry'; +import { TelemetryService } from '../../../telemetry'; import StatusView from '../../../views/statusView'; import ConnectionController from '../../../connectionController'; import EditDocumentCodeLensProvider from '../../../editors/editDocumentCodeLensProvider'; diff --git a/src/test/suite/editors/mongoDBDocumentService.test.ts b/src/test/suite/editors/mongoDBDocumentService.test.ts index 5f9be2f86..5c0530b7b 100644 --- a/src/test/suite/editors/mongoDBDocumentService.test.ts +++ b/src/test/suite/editors/mongoDBDocumentService.test.ts @@ -11,7 +11,7 @@ import MongoDBDocumentService from '../../../editors/mongoDBDocumentService'; import { StorageController } from '../../../storage'; import { StatusView } from '../../../views'; -import TelemetryService from '../../../telemetry'; +import { TelemetryService } from '../../../telemetry'; import { ExtensionContextStub } from '../stubs'; const expect = chai.expect; diff --git a/src/test/suite/editors/playgroundController.test.ts b/src/test/suite/editors/playgroundController.test.ts index eba4a2652..f13a79c53 100644 --- a/src/test/suite/editors/playgroundController.test.ts +++ b/src/test/suite/editors/playgroundController.test.ts @@ -14,7 +14,7 @@ import { PlaygroundController } from '../../../editors'; import PlaygroundResultProvider from '../../../editors/playgroundResultProvider'; import { StatusView } from '../../../views'; import { StorageController } from '../../../storage'; -import TelemetryService from '../../../telemetry'; +import { TelemetryService } from '../../../telemetry'; import { TEST_DATABASE_URI } from '../dbTestHelper'; import { ExtensionContextStub, LanguageServerControllerStub } from '../stubs'; import { mockTextEditor } from '../stubs'; diff --git a/src/test/suite/editors/playgroundResultProvider.test.ts b/src/test/suite/editors/playgroundResultProvider.test.ts index 25edda64f..6b7efc5ae 100644 --- a/src/test/suite/editors/playgroundResultProvider.test.ts +++ b/src/test/suite/editors/playgroundResultProvider.test.ts @@ -14,7 +14,7 @@ import EditDocumentCodeLensProvider from '../../../editors/editDocumentCodeLensP import PlaygroundResultProvider from '../../../editors/playgroundResultProvider'; import { StatusView } from '../../../views'; import { StorageController } from '../../../storage'; -import TelemetryService from '../../../telemetry'; +import { TelemetryService } from '../../../telemetry'; import { ExtensionContextStub } from '../stubs'; const expect = chai.expect; diff --git a/src/test/suite/language/languageServerController.test.ts b/src/test/suite/language/languageServerController.test.ts index a1b214791..00876ae62 100644 --- a/src/test/suite/language/languageServerController.test.ts +++ b/src/test/suite/language/languageServerController.test.ts @@ -18,7 +18,7 @@ import PlaygroundResultProvider from '../../../editors/playgroundResultProvider' import { StatusView } from '../../../views'; import { StorageController } from '../../../storage'; import { TEST_DATABASE_URI } from '../dbTestHelper'; -import TelemetryService from '../../../telemetry'; +import { TelemetryService } from '../../../telemetry'; import { ExtensionContextStub } from '../stubs'; import ExportToLanguageCodeLensProvider from '../../../editors/exportToLanguageCodeLensProvider'; diff --git a/src/test/suite/oidc.test.ts b/src/test/suite/oidc.test.ts index cfcfb3579..41aa63099 100644 --- a/src/test/suite/oidc.test.ts +++ b/src/test/suite/oidc.test.ts @@ -11,7 +11,7 @@ import { before, after, afterEach, beforeEach } from 'mocha'; import EventEmitter, { once } from 'events'; import { ExtensionContextStub } from './stubs'; import { StorageController } from '../../storage'; -import TelemetryService from '../../telemetry'; +import { TelemetryService } from '../../telemetry'; import ConnectionController from '../../connectionController'; import { StatusView } from '../../views'; import { waitFor } from './waitFor'; diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts index 47539d7f1..4b3c99c78 100644 --- a/src/test/suite/participant/participant.test.ts +++ b/src/test/suite/participant/participant.test.ts @@ -48,7 +48,7 @@ import type { SendMessageToParticipantOptions, } from '../../../participant/participantTypes'; import { DocumentSource } from '../../../documentSource'; -import TelemetryService from '../../../telemetry'; +import { TelemetryService } from '../../../telemetry'; // The Copilot's model in not available in tests, // therefore we need to mock its methods and returning values. diff --git a/src/test/suite/views/webviewController.test.ts b/src/test/suite/views/webviewController.test.ts index a88ace2bd..ed8c33ae1 100644 --- a/src/test/suite/views/webviewController.test.ts +++ b/src/test/suite/views/webviewController.test.ts @@ -10,7 +10,7 @@ import { mdbTestExtension } from '../stubbableMdbExtension'; import { MESSAGE_TYPES } from '../../../views/webview-app/extension-app-message-constants'; import { StatusView } from '../../../views'; import { StorageController } from '../../../storage'; -import TelemetryService from '../../../telemetry'; +import { TelemetryService } from '../../../telemetry'; import { ExtensionContextStub } from '../stubs'; import { TEST_DATABASE_URI } from '../dbTestHelper'; import WebviewController, { diff --git a/src/views/webviewController.ts b/src/views/webviewController.ts index 02c21080c..927b7648d 100644 --- a/src/views/webviewController.ts +++ b/src/views/webviewController.ts @@ -15,7 +15,7 @@ import { } from './webview-app/extension-app-message-constants'; import { openLink } from '../utils/linkHelper'; import type { StorageController } from '../storage'; -import type TelemetryService from '../telemetry'; +import type { TelemetryService } from '../telemetry'; import { getFeatureFlagsScript } from '../featureFlags'; import { ConnectionEditedTelemetryEvent,