[Text Analytics] Fix a bug for returning stats for analyze (Azure#14143)

This PR fixes several issues:

- when the user asks for statistics, they did not get it because the request parameter did not make it to the status endpoint. This PR fixes this issue by sending the `includeStatistics` parameter to the status endpoint
- removes statistics from the top-level response because the service does not return it. See issue:Azure#14139
- the LRO's onProgress function was broken because the input callback was only applied when the LRO finished which defeats the purpose
- the new core lib parameters (e.g. `onResponse`) were not being handled correctly
- updates the samples with the includeStatistics parameter

Author: deyaaeldeen

sdk/textanalytics/ai-text-analytics/CHANGELOG.md

@@ -9,6 +9,7 @@
 - `beginAnalyzeHealthcareEntities` entities now include `assertions` instead of `isNegated` which gives more context about the respective entity.
 - [Breaking] `beginAnalyzeHealthcareEntities` no longer returns `relatedEntities`.
 - `recognizePiiEntities` takes a new option, `categoriesFilter`, that specifies a list of Pii categories to return.
+- [Breaking] `statistics` is no longer part of `PagedAnalyzeBatchActionsResult`.
 
 ## 5.1.0-beta.4 (2021-02-10)
 

sdk/textanalytics/ai-text-analytics/README.md

@@ -500,7 +500,6 @@ main();
 
 ## Known Issues
 
-- Currently, the `beginAnalyze` API accepts `includeStatistics` in its options bag, a feature that was not yet supported by the service at the time of the current release. This feature is expected to be supported soon after the release.
 - `beginAnalyzeHealthcare` is still in gated preview and can not be used with AAD credentials. For more information, see (the Text Analytics for Health documentation)[https://docs.microsoft.com/en-us/azure/cognitive-services/text-analytics/how-tos/text-analytics-for-health?tabs=ner#request-access-to-the-public-preview].
 
 ## Troubleshooting

sdk/textanalytics/ai-text-analytics/recordings/browsers/aad_textanalyticsclient_lros_analyze/recording_statistics.json

@@ -318,7 +318,6 @@ export interface Opinion {
 
 // @public
 export interface PagedAnalyzeBatchActionsResult extends PagedAsyncIterableAnalyzeBatchActionsResult {
-    statistics?: TextDocumentBatchStatistics;
 }
 
 // @public

sdk/textanalytics/ai-text-analytics/recordings/node/aad_textanalyticsclient_lros_analyze/recording_statistics.js

@@ -33,7 +33,22 @@ async function main() {
     recognizePiiEntitiesActions: [{ modelVersion: "latest" }],
     extractKeyPhrasesActions: [{ modelVersion: "latest" }]
   };
-  const poller = await client.beginAnalyzeBatchActions(documents, actions);
+  const poller = await client.beginAnalyzeBatchActions(documents, actions, "en", {
+    includeStatistics: true
+  });
+  poller.onProgress(() => {
+    console.log(
+      `Number of actions still in progress: ${poller.getOperationState().actionsInProgressCount}`
+    );
+  });
+  console.log(
+    `The analyze batch actions operation was created on ${poller.getOperationState().createdOn}`
+  );
+  console.log(
+    `The analyze batch actions operation results will expire on ${
+      poller.getOperationState().expiresOn
+    }`
+  );
   const resultPages = await poller.pollUntilDone();
   for await (const page of resultPages) {
     const keyPhrasesAction = page.extractKeyPhrasesResults[0];
@@ -49,6 +64,8 @@ async function main() {
           console.error("\tError:", doc.error);
         }
       }
+      console.log("Action statistics: ");
+      console.log(JSON.stringify(keyPhrasesAction.results.statistics));
     }
 
     const entitiesAction = page.recognizeEntitiesResults[0];
@@ -64,6 +81,8 @@ async function main() {
           console.error("\tError:", doc.error);
         }
       }
+      console.log("Action statistics: ");
+      console.log(JSON.stringify(entitiesAction.results.statistics));
     }
 
     const piiEntitiesAction = page.recognizePiiEntitiesResults[0];
@@ -79,6 +98,8 @@ async function main() {
           console.error("\tError:", doc.error);
         }
       }
+      console.log("Action statistics: ");
+      console.log(JSON.stringify(piiEntitiesAction.results.statistics));
     }
   }
 }

sdk/textanalytics/ai-text-analytics/review/ai-text-analytics.api.md

@@ -33,12 +33,17 @@ export async function main() {
     recognizePiiEntitiesActions: [{ modelVersion: "latest" }],
     extractKeyPhrasesActions: [{ modelVersion: "latest" }]
   };
-  const poller = await client.beginAnalyzeBatchActions(documents, actions);
+  const poller = await client.beginAnalyzeBatchActions(documents, actions, "en", {
+    includeStatistics: true
+  });
   const resultPages = await poller.pollUntilDone();
+  poller.onProgress(() => {
+    console.log(
+      `Number of actions still in progress: ${poller.getOperationState().actionsInProgressCount}`
+    );
+  });
   console.log(
-    `The analyze batch actions operation created on ${
-      poller.getOperationState().createdOn
-    } finished process`
+    `The analyze batch actions operation created on ${poller.getOperationState().createdOn}`
   );
   console.log(
     `The analyze batch actions operation results will expire on ${
@@ -60,6 +65,8 @@ export async function main() {
           console.error("\tError:", doc.error);
         }
       }
+      console.log("Action statistics: ");
+      console.log(JSON.stringify(keyPhrasesAction.results.statistics));
     }
 
     const entitiesAction = page.recognizeEntitiesResults[0];
@@ -75,6 +82,8 @@ export async function main() {
           console.error("\tError:", doc.error);
         }
       }
+      console.log("Action statistics: ");
+      console.log(JSON.stringify(entitiesAction.results.statistics));
     }
 
     const piiEntitiesAction = page.recognizePiiEntitiesResults[0];
@@ -90,6 +99,8 @@ export async function main() {
           console.error("\tError:", doc.error);
         }
       }
+      console.log("Action statistics: ");
+      console.log(JSON.stringify(piiEntitiesAction.results.statistics));
     }
   }
 }

sdk/textanalytics/ai-text-analytics/samples/javascript/beginAnalyzeBatchActions.js

@@ -12,7 +12,6 @@ import {
   TasksStateTasksEntityRecognitionPiiTasksItem,
   TasksStateTasksEntityRecognitionTasksItem,
   TasksStateTasksKeyPhraseExtractionTasksItem,
-  TextDocumentBatchStatistics,
   TextDocumentInput
 } from "./generated/models";
 import {
@@ -184,12 +183,12 @@ export type PagedAsyncIterableAnalyzeBatchActionsResult = PagedAsyncIterableIter
  */
 export interface PagedAnalyzeBatchActionsResult
   extends PagedAsyncIterableAnalyzeBatchActionsResult {
-  /**
-   * Statistics about the input document batch and how it was processed
-   * by the service. This property will have a value when includeStatistics is set to true
-   * in the client call.
-   */
-  statistics?: TextDocumentBatchStatistics;
+  // /**
+  //  * Statistics about the input document batch and how it was processed
+  //  * by the service. This property will have a value when includeStatistics is set to true
+  //  * in the client call.
+  //  */
+  // statistics?: TextDocumentBatchStatistics;
 }
 
 /**

sdk/textanalytics/ai-text-analytics/samples/typescript/src/beginAnalyzeBatchActions.ts

@@ -6,6 +6,7 @@ import { OperationOptions } from "@azure/core-client";
 import { AbortSignalLike } from "@azure/abort-controller";
 
 import {
+  AnalyzeJobState,
   GeneratedClientAnalyzeResponse as BeginAnalyzeResponse,
   GeneratedClientAnalyzeStatusOptionalParams as AnalyzeBatchActionsOperationStatusOptions,
   JobManifestTasks as GeneratedActions,
@@ -101,6 +102,21 @@ export interface AnalyzeBatchActionsOperationState
   extends AnalysisPollOperationState<PagedAnalyzeBatchActionsResult>,
     AnalyzeBatchActionsOperationMetadata {}
 
+/**
+ * @internal
+ */
+function getMetaInfoFromResponse(response: AnalyzeJobState): AnalyzeBatchActionsOperationMetadata {
+  return {
+    createdOn: response.createdDateTime,
+    lastModifiedOn: response.lastUpdateDateTime,
+    expiresOn: response.expirationDateTime,
+    status: response.status,
+    actionsSucceededCount: response.tasks.completed,
+    actionsFailedCount: response.tasks.failed,
+    actionsInProgressCount: response.tasks.inProgress
+  };
+}
+
 /**
  * Class that represents a poller that waits for results of the analyze
  * operation.
@@ -218,19 +234,11 @@ export class BeginAnalyzeBatchActionsPollerOperation extends AnalysisPollOperati
           return {
             done: true,
             statistics: response.statistics,
-            operationMetdata: {
-              createdOn: response.createdDateTime,
-              lastModifiedOn: response.lastUpdateDateTime,
-              expiresOn: response.expirationDateTime,
-              status: response.status,
-              actionsSucceededCount: response.tasks.completed,
-              actionsFailedCount: response.tasks.failed,
-              actionsInProgressCount: response.tasks.inProgress
-            }
+            operationMetdata: getMetaInfoFromResponse(response)
           };
         }
       }
-      return { done: false };
+      return { done: false, operationMetdata: getMetaInfoFromResponse(response) };
     } catch (e) {
       span.setStatus({
         code: CanonicalCode.UNKNOWN,
@@ -311,18 +319,25 @@ export class BeginAnalyzeBatchActionsPollerOperation extends AnalysisPollOperati
     state.actionsInProgressCount = operationStatus.operationMetdata?.actionsInProgressCount;
 
     if (!state.isCompleted && operationStatus.done) {
-      if (typeof options.fireProgress === "function") {
-        options.fireProgress(state);
-      }
       const pagedIterator = this.listAnalyzeBatchActionsResults(state.operationId!, {
         abortSignal: this.options.abortSignal,
-        tracingOptions: this.options.tracingOptions
-      });
-      state.result = Object.assign(pagedIterator, {
-        statistics: operationStatus.statistics
+        tracingOptions: this.options.tracingOptions,
+        includeStatistics: this.options.includeStatistics,
+        onResponse: this.options.onResponse,
+        serializerOptions: this.options.serializerOptions
       });
+      // Attach stats if the service starts to return them
+      // https://github.com/Azure/azure-sdk-for-js/issues/14139
+      // state.result = Object.assign(pagedIterator, {
+      //   statistics: operationStatus.statistics
+      // });
+      state.result = pagedIterator;
       state.isCompleted = true;
     }
+
+    if (typeof options.fireProgress === "function") {
+      options.fireProgress(state);
+    }
     return this;
   }
 

sdk/textanalytics/ai-text-analytics/src/analyzeBatchActionsResult.ts

@@ -48,6 +48,7 @@ import { textAnalyticsAzureKeyCredentialPolicy } from "./azureKeyCredentialPolic
 import {
   AddParamsToTask,
   addStrEncodingParam,
+  compose,
   handleInvalidDocumentBatch,
   setModelVersionParam,
   setStrEncodingParam,
@@ -904,7 +905,9 @@ export class TextAnalyticsClient {
       analysisOptions: {
         requestOptions: realOptions.requestOptions,
         tracingOptions: realOptions.tracingOptions,
-        abortSignal: realOptions.abortSignal
+        abortSignal: realOptions.abortSignal,
+        onResponse: realOptions.onResponse,
+        serializerOptions: realOptions.serializerOptions
       },
       updateIntervalInMs: realOptions.updateIntervalInMs,
       resumeFrom: realOptions.resumeFrom,
@@ -974,7 +977,9 @@ export class TextAnalyticsClient {
       analysisOptions: {
         requestOptions: realOptions.requestOptions,
         tracingOptions: realOptions.tracingOptions,
-        abortSignal: realOptions.abortSignal
+        abortSignal: realOptions.abortSignal,
+        onResponse: realOptions.onResponse,
+        serializerOptions: realOptions.serializerOptions
       },
       includeStatistics: realOptions.includeStatistics,
       updateIntervalInMs: realOptions.updateIntervalInMs,
@@ -986,13 +991,6 @@ export class TextAnalyticsClient {
   }
 }
 
-/**
- * @internal
- */
-function compose<T1, T2, T3>(fn1: (x: T1) => T2, fn2: (y: T2) => T3): (x: T1) => T3 {
-  return (value: T1) => fn2(fn1(value));
-}
-
 /**
  * @internal
  */
@@ -1068,7 +1066,9 @@ function makeAnalyzeSentimentOptionsModel(
     modelVersion: params.modelVersion,
     requestOptions: params.requestOptions,
     stringIndexType: params.stringIndexType,
-    tracingOptions: params.tracingOptions
+    tracingOptions: params.tracingOptions,
+    onResponse: params.onResponse,
+    serializerOptions: params.serializerOptions
   };
 }
 
@@ -1088,6 +1088,8 @@ function makePiiEntitiesOptionsModel(
     requestOptions: params.requestOptions,
     stringIndexType: params.stringIndexType,
     tracingOptions: params.tracingOptions,
-    piiCategories: params.categoriesFilter
+    piiCategories: params.categoriesFilter,
+    onResponse: params.onResponse,
+    serializerOptions: params.serializerOptions
   };
 }