Skip to content

Commit 77b0b6a

Browse files
witemple-msftwitemple-msft
authored
[perf-ai-form-recognizer] First stab at an LRO perf test. (Azure#14013)
* [perf-ai-form-recognizer] First stab at an LRO perf test. * WIP * Addressed some feedback * Moved some stuff around, addressed more feedback. * Fixed some compiler errors
1 parent 10395be commit 77b0b6a

File tree

8 files changed

+536
-333
lines changed

8 files changed

+536
-333
lines changed

‎common/config/rush/pnpm-lock.yaml‎

Lines changed: 335 additions & 333 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎rush.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -647,6 +647,11 @@
647647
"projectFolder": "sdk/digitaltwins/digital-twins-core",
648648
"versionPolicyName": "client"
649649
},
650+
{
651+
"packageName": "@azure-tests/perf-ai-form-recognizer",
652+
"projectFolder": "sdk/formrecognizer/perf-tests/ai-form-recognizer",
653+
"versionPolicyName": "test"
654+
},
650655
{
651656
"packageName": "@azure-tests/perf-eventgrid",
652657
"projectFolder": "sdk/eventgrid/perf-tests/eventgrid",

‎sdk/formrecognizer/perf-tests/ai-form-recognizer/README.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
### Guide
2+
3+
1. Build the ai-form-recognizer perf tests package `rush build -t @azure-tests/perf-ai-form-recognizer`.
4+
2. Copy the `sample.env` file and name it as `.env`.
5+
3. Create a cognitive services account and populate the `.env` file with the relevant credentials.
6+
4. Run the tests as follows
7+
8+
- custom model recognition
9+
- `npm run perf-test:node -- CustomModelRecognitionTest --warmup 2 --duration 7 --iterations 2 --parallel 2`
10+
11+
Be wary of low API rate limits. Increasing the "parallel" parameter can cause the performance tests to exceed the allowed request rate.

‎sdk/formrecognizer/perf-tests/ai-form-recognizer/package.json‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
{
2+
"name": "@azure-tests/perf-ai-form-recognizer",
3+
"version": "1.0.0",
4+
"private": true,
5+
"description": "Performance Tests for @azure/ai-form-recognizer",
6+
"main": "",
7+
"keywords": [],
8+
"author": "",
9+
"license": "ISC",
10+
"dependencies": {
11+
"@azure/ai-form-recognizer": "3.1.0-beta.3",
12+
"@azure/identity": "^1.1.0",
13+
"@azure/test-utils-perfstress": "^1.0.0",
14+
"dotenv": "^8.2.0",
15+
"tslib": "^2.0.0"
16+
},
17+
"devDependencies": {
18+
"@types/node": "^8.0.0",
19+
"rimraf": "^3.0.0",
20+
"ts-node": "^9.0.0",
21+
"typescript": "~4.2.0",
22+
"prettier": "^1.16.4"
23+
},
24+
"scripts": {
25+
"perf-test:node": "ts-node test/index.spec.ts",
26+
"audit": "node ../../../common/scripts/rush-audit.js && rimraf node_modules package-lock.json && npm i --package-lock-only 2>&1 && npm audit",
27+
"build": "tsc -p .",
28+
"build:samples": "echo skipped",
29+
"build:test": "echo skipped",
30+
"check-format": "prettier --list-different --config ../../../../.prettierrc.json --ignore-path ../../../.prettierignore \"src/**/*.ts\" \"test/**/*.ts\" \"*.{js,json}\"",
31+
"clean": "rimraf dist dist-esm test-dist typings *.tgz *.log",
32+
"format": "prettier --write --config ../../../../.prettierrc.json --ignore-path ../../../.prettierignore \"src/**/*.ts\" \"test/**/*.ts\" \"*.{js,json}\"",
33+
"integration-test:browser": "echo skipped",
34+
"integration-test:node": "echo skipped",
35+
"integration-test": "echo skipped",
36+
"lint:fix": "eslint package.json src test --ext .ts --fix --fix-type [problem,suggestion]",
37+
"lint": "eslint package.json src test --ext .ts -f html -o perf-ai-form-recognizer-lintReport.html || exit 0",
38+
"pack": "npm pack 2>&1",
39+
"prebuild": "npm run clean",
40+
"unit-test:browser": "echo skipped",
41+
"unit-test:node": "echo skipped",
42+
"unit-test": "echo skipped",
43+
"test:browser": "echo skipped",
44+
"test:node": "echo skipped",
45+
"test": "echo skipped"
46+
}
47+
}

‎sdk/formrecognizer/perf-tests/ai-form-recognizer/sample.env‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Used to authenticate with the Cognitive Services endpoint
2+
FORM_RECOGNIZER_ENDPOINT="https://<resource name>.cognitiveservies.azure.com/"
3+
FORM_RECOGNIZER_API_KEY="<form recognizer api key>"
4+
5+
# Model training information: the first variable is used to create a model during
6+
# test setup, and the second refers to the document that we send to the service
7+
# to use for recognition
8+
FORM_RECOGNIZER_TRAINING_CONTAINER_SAS_URL="<sas url to a container of training documents>"
9+
FORM_RECOGNIZER_TEST_DOCUMENT_URL="<url to a test document to send>"
10+
11+
# As an alternative to FORM_RECOGNIZER_API_KEY, you may provide Azure Active
12+
# Directory credentials below.
13+
AZURE_TENANT_ID="<azure tenant id>"
14+
AZURE_CLIENT_ID="<azure client id>"
15+
AZURE_CLIENT_SECRET="<azure client secret>"

‎sdk/formrecognizer/perf-tests/ai-form-recognizer/test/custom.spec.ts‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT license.
3+
4+
import {
5+
PerfStressOptionDictionary,
6+
PerfStressTest,
7+
getEnvVar
8+
} from "@azure/test-utils-perfstress";
9+
import {
10+
AzureKeyCredential,
11+
BeginRecognizeCustomFormsOptions,
12+
CustomFormModel,
13+
FormRecognizerClient,
14+
FormTrainingClient
15+
} from "@azure/ai-form-recognizer";
16+
import { DefaultAzureCredential, TokenCredential } from "@azure/identity";
17+
18+
function unreachable(message?: string): never {
19+
throw new Error(message ?? "Unreachable Exception.");
20+
}
21+
22+
export class CustomModelRecognitionTest extends PerfStressTest<BeginRecognizeCustomFormsOptions> {
23+
public options: PerfStressOptionDictionary<BeginRecognizeCustomFormsOptions> = {
24+
updateIntervalInMs: {
25+
required: false,
26+
description: "Polling interval in milliseconds",
27+
shortName: "u",
28+
longName: "update-interval",
29+
defaultValue: 5000
30+
}
31+
};
32+
33+
/**
34+
* Not thrilled about this, but `globalSetup` only runs once overall, while `setup`
35+
* shouldn't have to train the model every time.
36+
*/
37+
static sessionModel: CustomFormModel | undefined = undefined;
38+
39+
private recognizerClient: FormRecognizerClient;
40+
private trainingClient: FormTrainingClient;
41+
42+
private documentUrl: string;
43+
44+
constructor() {
45+
super();
46+
47+
let credential: TokenCredential | AzureKeyCredential;
48+
49+
try {
50+
credential = new DefaultAzureCredential();
51+
} catch {
52+
credential = new AzureKeyCredential(getEnvVar("FORM_RECOGNIZER_API_KEY"));
53+
}
54+
55+
const endpoint = getEnvVar("FORM_RECOGNIZER_ENDPOINT");
56+
57+
this.trainingClient = new FormTrainingClient(endpoint, credential);
58+
this.recognizerClient = new FormRecognizerClient(endpoint, credential);
59+
60+
this.documentUrl = getEnvVar("FORM_RECOGNIZER_TEST_DOCUMENT_URL");
61+
}
62+
63+
public async globalSetup() {
64+
const trainingContainerSasUrl = getEnvVar("FORM_RECOGNIZER_TRAINING_CONTAINER_SAS_URL");
65+
66+
try {
67+
const poller = await this.trainingClient.beginTraining(trainingContainerSasUrl, true);
68+
69+
CustomModelRecognitionTest.sessionModel = await poller.pollUntilDone();
70+
71+
console.log(`Trained custom model: ${CustomModelRecognitionTest.sessionModel.modelId}`);
72+
} catch (ex) {
73+
console.trace(ex);
74+
throw ex;
75+
}
76+
}
77+
78+
public async globalCleanup() {
79+
const modelId = CustomModelRecognitionTest.sessionModel?.modelId;
80+
if (modelId) {
81+
console.log(`Deleting ${modelId}`);
82+
await this.trainingClient.deleteModel(modelId);
83+
}
84+
}
85+
86+
async runAsync(): Promise<void> {
87+
const modelId = CustomModelRecognitionTest.sessionModel?.modelId;
88+
if (!modelId) {
89+
return unreachable("Failed to initialize model.");
90+
}
91+
92+
const poller = await this.recognizerClient.beginRecognizeCustomFormsFromUrl(
93+
modelId,
94+
this.documentUrl,
95+
{
96+
updateIntervalInMs: this.parsedOptions.updateIntervalInMs?.value
97+
}
98+
);
99+
100+
await poller.pollUntilDone();
101+
}
102+
}

‎sdk/formrecognizer/perf-tests/ai-form-recognizer/test/index.spec.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT license.
3+
4+
import { PerfStressProgram, selectPerfStressTest } from "@azure/test-utils-perfstress";
5+
import { CustomModelRecognitionTest } from "./custom.spec";
6+
7+
import dotenv from "dotenv";
8+
dotenv.config();
9+
10+
console.log("=== Starting the perfStress test ===");
11+
12+
const perfStressProgram = new PerfStressProgram(selectPerfStressTest([CustomModelRecognitionTest]));
13+
14+
perfStressProgram.run();

‎sdk/formrecognizer/perf-tests/ai-form-recognizer/tsconfig.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"extends": "../../../../tsconfig.package",
3+
"compilerOptions": {
4+
"outDir": "./dist-esm"
5+
},
6+
"include": ["./test/**/*.ts"]
7+
}

0 commit comments

Comments
 (0)