Skip to content

Commit ba6c4aa

Browse files
hmoazamhmoazam
committed
Cosmos integration test added, and test-env README updated. TODO: Update LIMITATIONS, add integration test notebooks, and add integration test for additional inputs + cosmos
1 parent c73398b commit ba6c4aa

File tree

4 files changed

+171
-1
lines changed

4 files changed

+171
-1
lines changed

‎LIMITATIONS.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ The solution accelerator supports a limited set of data sources to be ingested i
1313
* [Azure MySQL](#azure-mysql)
1414
* [PostgreSQL](#postgresql)
1515
* [Azure Data Explorer](#azure-data-explorer)
16+
* [Azure Cosmos DB]()
1617
* [Other Data Sources and Limitations](#other-data-sources-and-limitations)
1718
* [Column Level Mapping Supported Sources](#column-level-mapping-supported-sources)
1819

‎function-app/adb-to-purview/tests/unit-tests/Function.Domain/Helpers/Parser/QnParserTests.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,10 @@ public QnParserTests()
148148
[InlineData("azurekusto://qpll4l5hchczm.eastus2.kusto.windows.net/database01",
149149
"table01",
150150
"https://qpll4l5hchczm.eastus2.kusto.windows.net/database01/table01")]
151+
// Cosmos
152+
[InlineData("azurecosmos://6ch4pkm5tpniq.documents.azure.com/dbs/myDatabase",
153+
"/colls/yourContainer",
154+
"https://6ch4pkm5tpniq.documents.azure.com/dbs/mydatabase/colls/yourcontainer")]
151155

152156
public void GetIdentifiers_OlSource_ReturnsPurviewIdentifier(string nameSpace, string name, string expectedResult)
153157
{

‎tests/environment/README.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ Add Key Vault Secrets
5050
* `azurekusto-appsecret`
5151
* `azurekusto-uri`
5252

53+
* `azurecosmos-endpoint`
54+
* `azurecosmos-key`
5355
* Update SQL Db and Synapse Server with AAD Admin
5456
* Add Service Principal for Databricks to connect to SQL sources
5557
* Assign the Service Principal admin role on the ADX cluster. [Guide](https://learn.microsoft.com/en-us/azure/data-explorer/provision-azure-ad-app#grant-the-service-principal-access-to-an-azure-data-explorer-database)
@@ -63,9 +65,12 @@ Set the following system environments:
6365
Install the version of the [kusto spark connector](https://github.com/Azure/azure-kusto-spark) that matches the cluster Scala and Spark versions from Maven Central.
6466

6567
Upload notebooks in `./tests/integration/spark-apps/notebooks/` to dbfs' `/Shared/examples/`
66-
6768
* Manually for now. TODO: Automate this in Python
6869

70+
Install the following libraries on the compute cluster (versions to match the Spark and Scala versions of the cluster) (TODO: Automate):
71+
* Cosmos spark connector
72+
73+
6974
Compile the following apps and upload them to `/dbfs/FileStore/testcases/`
7075

7176
* `./tests/integration/spark-apps/jarjobs/abfssInAbfssOut/` with `./gradlew build`

‎tests/environment/sources/cosmos.bicep‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
@description('Azure Cosmos DB account name, max length 44 characters')
2+
param accountName string = uniqueString('cosmos', resourceGroup().id)
3+
4+
@description('Location for the Azure Cosmos DB account.')
5+
param location string = resourceGroup().location
6+
7+
@allowed([
8+
'Eventual'
9+
'ConsistentPrefix'
10+
'Session'
11+
'BoundedStaleness'
12+
'Strong'
13+
])
14+
@description('The default consistency level of the Cosmos DB account.')
15+
param defaultConsistencyLevel string = 'Session'
16+
17+
@minValue(10)
18+
@maxValue(2147483647)
19+
@description('Max stale requests. Required for BoundedStaleness. Valid ranges, Single Region: 10 to 2147483647. Multi Region: 100000 to 2147483647.')
20+
param maxStalenessPrefix int = 100000
21+
22+
@minValue(5)
23+
@maxValue(86400)
24+
@description('Max lag time (minutes). Required for BoundedStaleness. Valid ranges, Single Region: 5 to 84600. Multi Region: 300 to 86400.')
25+
param maxIntervalInSeconds int = 300
26+
27+
@allowed([
28+
true
29+
false
30+
])
31+
@description('Enable system managed failover for regions')
32+
param systemManagedFailover bool = false
33+
34+
@description('The name for the database')
35+
param databaseName string = 'myDatabase'
36+
37+
@description('The name for the input container')
38+
param inContainerName string = 'myContainer'
39+
40+
@description('The name for the output container')
41+
param outContainerName string = 'yourContainer'
42+
43+
@minValue(400)
44+
@maxValue(1000000)
45+
@description('The throughput for the container')
46+
param throughput int = 400
47+
48+
var consistencyPolicy = {
49+
Eventual: {
50+
defaultConsistencyLevel: 'Eventual'
51+
}
52+
ConsistentPrefix: {
53+
defaultConsistencyLevel: 'ConsistentPrefix'
54+
}
55+
Session: {
56+
defaultConsistencyLevel: 'Session'
57+
}
58+
BoundedStaleness: {
59+
defaultConsistencyLevel: 'BoundedStaleness'
60+
maxStalenessPrefix: maxStalenessPrefix
61+
maxIntervalInSeconds: maxIntervalInSeconds
62+
}
63+
Strong: {
64+
defaultConsistencyLevel: 'Strong'
65+
}
66+
}
67+
var locations = [
68+
{
69+
locationName: location
70+
failoverPriority: 0
71+
isZoneRedundant: false
72+
}
73+
]
74+
75+
resource account 'Microsoft.DocumentDB/databaseAccounts@2022-05-15' = {
76+
name: toLower(accountName)
77+
location: location
78+
kind: 'GlobalDocumentDB'
79+
properties: {
80+
consistencyPolicy: consistencyPolicy[defaultConsistencyLevel]
81+
locations: locations
82+
databaseAccountOfferType: 'Standard'
83+
enableAutomaticFailover: systemManagedFailover
84+
}
85+
}
86+
87+
resource database 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases@2022-05-15' = {
88+
name: '${account.name}/${databaseName}'
89+
properties: {
90+
resource: {
91+
id: databaseName
92+
}
93+
}
94+
}
95+
96+
resource container 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers@2022-05-15' = {
97+
name: '${database.name}/${inContainerName}'
98+
properties: {
99+
resource: {
100+
id: inContainerName
101+
partitionKey: {
102+
paths: [
103+
'/country'
104+
]
105+
kind: 'Hash'
106+
}
107+
indexingPolicy: {
108+
indexingMode: 'consistent'
109+
automatic: true
110+
includedPaths: [
111+
{
112+
path: '/*'
113+
}
114+
]
115+
excludedPaths: [
116+
{
117+
path: '/\'_etag\'/?'
118+
}
119+
]
120+
}
121+
defaultTtl: 86400
122+
}
123+
options: {
124+
throughput: throughput
125+
}
126+
}
127+
}
128+
129+
resource container2 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers@2022-05-15' = {
130+
name: '${database.name}/${outContainerName}'
131+
properties: {
132+
resource: {
133+
id: outContainerName
134+
partitionKey: {
135+
paths: [
136+
'/country'
137+
]
138+
kind: 'Hash'
139+
}
140+
indexingPolicy: {
141+
indexingMode: 'consistent'
142+
automatic: true
143+
includedPaths: [
144+
{
145+
path: '/*'
146+
}
147+
]
148+
excludedPaths: [
149+
{
150+
path: '/\'_etag\'/?'
151+
}
152+
]
153+
}
154+
defaultTtl: 86400
155+
}
156+
options: {
157+
throughput: throughput
158+
}
159+
}
160+
}

0 commit comments

Comments
 (0)