Skip to content

Commit c0a7a8d

Browse files
wangbwnBoWangzi1
wangbwn
and
BoWangzi1
authored
Add DataFactory Test Case (Azure#39756)
* Add unit test * Update copy data task test * Update bug * Add JsonPathSanitizers * Add DataFactoryElement Tests * Delete DataFactoryElementTests * Fix get sessionrecording.generateassetname * fix not use property * add test case * Update assets.json * Fix a bug * Fix utc time. --------- Co-authored-by: Bo Wang (Shanghai Wicresoft Co Ltd) <v-bw@microsoft.com>
1 parent c1e7924 commit c0a7a8d

10 files changed

+17852
-84
lines changed

‎sdk/datafactory/Azure.ResourceManager.DataFactory/assets.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"AssetsRepo": "Azure/azure-sdk-assets",
33
"AssetsRepoPrefixPath": "net",
44
"TagPrefix": "net/datafactory/Azure.ResourceManager.DataFactory",
5-
"Tag": "net/datafactory/Azure.ResourceManager.DataFactory_2ffeead585"
5+
"Tag": "net/datafactory/Azure.ResourceManager.DataFactory_d20fc87519"
66
}

‎sdk/datafactory/Azure.ResourceManager.DataFactory/tests/DataFactoryElementTestCollection.cs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Linq;
7+
using System.Text;
8+
using System.Text.Json.Serialization;
9+
using System.Threading.Tasks;
10+
using Azure.Core.Expressions.DataFactory;
11+
using Azure.ResourceManager.DataFactory.Models;
12+
13+
namespace Azure.ResourceManager.DataFactory.Tests
14+
{
15+
public class DataFactoryElementTestCollection
16+
{
17+
[JsonPropertyName("id")]
18+
public string Id { get; set; }
19+
20+
[JsonPropertyName("name")]
21+
public string Name { get; set; }
22+
23+
[JsonPropertyName("type")]
24+
public string Type { get; set; }
25+
26+
[JsonPropertyName("properties")]
27+
public Properties JsonProperties { get; set; }
28+
29+
public class Properties
30+
{
31+
[JsonPropertyName("type")]
32+
public string Type { get; set; }
33+
34+
[JsonPropertyName("typeProperties")]
35+
public TypeProperties JsonTypeProperties { get; set; }
36+
37+
[JsonPropertyName("schema")]
38+
public DataFactoryElement<IList<DatasetSchemaDataElement>> Schema { get; set; }
39+
40+
[JsonPropertyName("linkedServiceName")]
41+
public DataFactoryLinkedServiceReference LinkedServiceName { get; set; }
42+
}
43+
44+
public class TypeProperties
45+
{
46+
[JsonPropertyName("connectionString")]
47+
public DataFactoryElement<string> ConnectionString { get; set; }
48+
49+
[JsonPropertyName("location")]
50+
public Location Location { get; set; }
51+
52+
[JsonPropertyName("columnDelimiter")]
53+
public string ColumnDelimiter { get; set; }
54+
55+
[JsonPropertyName("firstRowAsHeader")]
56+
public bool FirstRowAsHeader { get; set; }
57+
}
58+
59+
public class Location
60+
{
61+
[JsonPropertyName("container")]
62+
public DataFactoryElement<string> Container { get; set; }
63+
64+
[JsonPropertyName("type")]
65+
public DataFactoryElement<string> Type { get; set; }
66+
67+
[JsonPropertyName("fileName")]
68+
public DataFactoryElement<string> FileName { get; set; }
69+
}
70+
}
71+
}

‎sdk/datafactory/Azure.ResourceManager.DataFactory/tests/Scenario/DataFactoryCopyDataTask.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
33

4+
using System;
45
using System.Threading.Tasks;
56
using Azure.Core;
67
using Azure.Core.TestFramework;

‎sdk/datafactory/Azure.ResourceManager.DataFactory/tests/Scenario/DataFactoryDataFlowTests.cs‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System.Threading.Tasks;
5+
using Azure.Core;
6+
using Azure.Core.Expressions.DataFactory;
7+
using Azure.Core.TestFramework;
8+
using Azure.ResourceManager.DataFactory.Models;
9+
using Azure.ResourceManager.Resources;
10+
using NUnit.Framework;
11+
12+
namespace Azure.ResourceManager.DataFactory.Tests.Scenario
13+
{
14+
internal class DataFactoryDataFlowTests : DataFactoryManagementTestBase
15+
{
16+
private ResourceIdentifier _resourceGroupIdentifier;
17+
private ResourceGroupResource _resourceGroup;
18+
private DataFactoryResource _dataFactory;
19+
public DataFactoryDataFlowTests(bool isAsync) : base(isAsync)
20+
{
21+
}
22+
23+
[OneTimeSetUp]
24+
public async Task GlobalSetup()
25+
{
26+
var rgName = SessionRecording.GenerateAssetName("DataFactory-RG-");
27+
var storageAccountName = SessionRecording.GenerateAssetName("datafactory");
28+
if (Mode == RecordedTestMode.Playback)
29+
{
30+
_resourceGroupIdentifier = ResourceGroupResource.CreateResourceIdentifier(SessionRecording.GetVariable("SUBSCRIPTION_ID", null), rgName);
31+
}
32+
else
33+
{
34+
using (SessionRecording.DisableRecording())
35+
{
36+
var subscription = await GlobalClient.GetDefaultSubscriptionAsync();
37+
var rgLro = await subscription.GetResourceGroups().CreateOrUpdateAsync(WaitUntil.Completed, rgName, new ResourceGroupData(AzureLocation.WestUS2));
38+
_resourceGroupIdentifier = rgLro.Value.Data.Id;
39+
}
40+
}
41+
await StopSessionRecordingAsync();
42+
}
43+
44+
[SetUp]
45+
public async Task TestSetUp()
46+
{
47+
string dataFactoryName = Recording.GenerateAssetName("DataFactory-");
48+
_resourceGroup = Client.GetResourceGroupResource(_resourceGroupIdentifier);
49+
_dataFactory = await CreateDataFactory(_resourceGroup, dataFactoryName);
50+
}
51+
52+
[TearDown]
53+
public async Task TestCaseDoneTearDown()
54+
{
55+
if (Mode == RecordedTestMode.Playback)
56+
{
57+
return;
58+
}
59+
try
60+
{
61+
using (Recording.DisableRecording())
62+
{
63+
await foreach (var dataFactoryResource in _resourceGroup.GetDataFactories().GetAllAsync())
64+
{
65+
await dataFactoryResource.DeleteAsync(WaitUntil.Completed);
66+
}
67+
}
68+
}
69+
catch (RequestFailedException ex) when (ex.Status == 404)
70+
{
71+
}
72+
}
73+
74+
private async Task<DataFactoryDatasetResource> CreateDefaultAzureSqlDatabaseDataset(string linkedServiceName, string datasetName)
75+
{
76+
DataFactoryLinkedServiceData lkSqlSource = new DataFactoryLinkedServiceData(new AzureSqlDatabaseLinkedService("Server=tcp:myServerAddress.database.windows.net,1433;Database=myDataBase;User ID=myUsername;Password=myPassword;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"));
77+
await _dataFactory.GetDataFactoryLinkedServices().CreateOrUpdateAsync(WaitUntil.Completed, linkedServiceName, lkSqlSource);
78+
79+
DataFactoryDatasetData data = new DataFactoryDatasetData(new AzureSqlTableDataset(new DataFactoryLinkedServiceReference(DataFactoryLinkedServiceReferenceType.LinkedServiceReference, linkedServiceName)));
80+
var result = await _dataFactory.GetDataFactoryDatasets().CreateOrUpdateAsync(WaitUntil.Completed, datasetName, data);
81+
return result.Value;
82+
}
83+
84+
[Test]
85+
[RecordedTest]
86+
public async Task DataFlow_ExecuteDataFlow()
87+
{
88+
string dataFlowName = Recording.GenerateAssetName("task-");
89+
string linkedServiceSourceName = Recording.GenerateAssetName("linkedService_");
90+
string linkedServiceSinkName = Recording.GenerateAssetName("linkedService_");
91+
string datasetSourceName1 = Recording.GenerateAssetName("dataset_");
92+
string datasetSourceName2 = Recording.GenerateAssetName("dataset_");
93+
string datasetSinkName1 = Recording.GenerateAssetName("dataset_");
94+
string datasetSinkName2 = Recording.GenerateAssetName("dataset_");
95+
96+
await CreateDefaultAzureSqlDatabaseDataset(linkedServiceSourceName, datasetSourceName1);
97+
await CreateDefaultAzureSqlDatabaseDataset(linkedServiceSinkName, datasetSourceName2);
98+
await CreateDefaultAzureSqlDatabaseDataset(linkedServiceSinkName, datasetSinkName1);
99+
await CreateDefaultAzureSqlDatabaseDataset(linkedServiceSinkName, datasetSinkName2);
100+
101+
DataFactoryDataFlowData mappingDataFlowForScriptLines = new DataFactoryDataFlowData(new DataFactoryMappingDataFlowProperties()
102+
{
103+
Sources =
104+
{
105+
new DataFlowSource(datasetSourceName1)
106+
{
107+
Dataset = new DatasetReference(DatasetReferenceType.DatasetReference,datasetSourceName1)
108+
},
109+
new DataFlowSource(datasetSourceName2)
110+
{
111+
Dataset = new DatasetReference(DatasetReferenceType.DatasetReference,datasetSourceName2)
112+
}
113+
},
114+
Sinks =
115+
{
116+
new DataFlowSink(datasetSinkName1)
117+
{
118+
Dataset = new DatasetReference(DatasetReferenceType.DatasetReference,datasetSinkName1)
119+
},
120+
new DataFlowSink(datasetSinkName2)
121+
{
122+
Dataset = new DatasetReference(DatasetReferenceType.DatasetReference,datasetSinkName2)
123+
}
124+
},
125+
Script = "fake some script",
126+
ScriptLines =
127+
{
128+
"some script1",
129+
"some script2"
130+
}
131+
});
132+
var resultForScriptLines = await _dataFactory.GetDataFactoryDataFlows().CreateOrUpdateAsync(WaitUntil.Completed, dataFlowName, mappingDataFlowForScriptLines);
133+
Assert.NotNull(resultForScriptLines.Value.Id);
134+
135+
DataFactoryDataFlowData mappingDataFlowForScript = new DataFactoryDataFlowData(new DataFactoryMappingDataFlowProperties()
136+
{
137+
Sources =
138+
{
139+
new DataFlowSource(datasetSourceName1)
140+
{
141+
Dataset = new DatasetReference(DatasetReferenceType.DatasetReference,datasetSourceName1)
142+
},
143+
new DataFlowSource(datasetSourceName2)
144+
{
145+
Dataset = new DatasetReference(DatasetReferenceType.DatasetReference,datasetSourceName2)
146+
}
147+
},
148+
Sinks =
149+
{
150+
new DataFlowSink(datasetSinkName1)
151+
{
152+
Dataset = new DatasetReference(DatasetReferenceType.DatasetReference,datasetSinkName1)
153+
},
154+
new DataFlowSink(datasetSinkName2)
155+
{
156+
Dataset = new DatasetReference(DatasetReferenceType.DatasetReference,datasetSinkName2)
157+
}
158+
},
159+
Script = "fake some script"
160+
});
161+
var resultForScript = await _dataFactory.GetDataFactoryDataFlows().CreateOrUpdateAsync(WaitUntil.Completed, dataFlowName, mappingDataFlowForScript);
162+
Assert.NotNull(resultForScript.Value.Id);
163+
}
164+
165+
[Test]
166+
[RecordedTest]
167+
public async Task Pipeline_ExecuteWarnglingDataflow_Queries()
168+
{
169+
string taskPowerQueryName = "powerquery1";
170+
string linkedServiceSourceName = Recording.GenerateAssetName("linkedService_");
171+
string datasetSourceName = "DS_AzureSqlDatabase1";
172+
string datasetSinkName = "DS_AzureSqlDatabase2";
173+
174+
await CreateDefaultAzureSqlDatabaseDataset(datasetSourceName, datasetSourceName);
175+
await CreateDefaultAzureSqlDatabaseDataset(linkedServiceSourceName, datasetSinkName);
176+
177+
DataFactoryDataFlowData mapping = new DataFactoryDataFlowData(new DataFactoryWranglingDataFlowProperties()
178+
{
179+
Sources =
180+
{
181+
new PowerQuerySource(datasetSourceName)
182+
{
183+
Dataset = new DatasetReference(DatasetReferenceType.DatasetReference,datasetSourceName),
184+
Script = "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tisolationLevel: 'READ_UNCOMMITTED',\n\tformat: 'table') ~> DS_AzureSqlDatabase1"
185+
},
186+
new PowerQuerySource(datasetSinkName)
187+
{
188+
Dataset = new DatasetReference(DatasetReferenceType.DatasetReference,datasetSinkName),
189+
Script = "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tisolationLevel: 'READ_UNCOMMITTED',\n\tformat: 'table') ~> DS_AzureSqlDatabase1"
190+
}
191+
},
192+
Script = "section Section1;\r\nshared DS_AzureSqlDatabase1 = let AdfDoc = Sql.Database(\"**********\", \"**********\", [CreateNavigationProperties = false]), InputTable = AdfDoc{[Schema=\"undefined\",Item=\"undefined\"]}[Data] in InputTable;\r\nshared UserQuery = let Source = #\"DS_AzureSqlDatabase1\" in Source;\r\n",
193+
DocumentLocale = "de-DE"
194+
});
195+
var result = await _dataFactory.GetDataFactoryDataFlows().CreateOrUpdateAsync(WaitUntil.Completed, taskPowerQueryName, mapping);
196+
Assert.NotNull(result.Value.Id);
197+
}
198+
}
199+
}

0 commit comments

Comments
 (0)