Skip to content

Commit 5ba00a7

Browse files
vidai-msftvidai-msft
authored
Fixed bugs for test coverage (Azure#20382)
* Added debug info for test coverage * Updated test coverage to fix regex issue * Updated databricks * Reverted databricks change * Changed the logic on retrieving the repo base folder * Load Az.Accounts under Debug folder for Databricks module. Testing purpose only in CI * Modified test coverage to support specifying the data location * Added properties StartDateTime and EndDateTime for test coverage raw data * Updated the fallback folder from .azure to .Azure * Changed to use resources string to get azure profile folder
1 parent 593f68d commit 5ba00a7

File tree

6 files changed

+110
-86
lines changed

6 files changed

+110
-86
lines changed

‎.azure-pipelines/powershell-core.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ variables:
1818
TestTimeoutInMinutes: 180
1919
BuildAzPredictor: false
2020
EnableTestCoverage: true
21+
TestCoverageLocation: $(Build.SourcesDirectory)/artifacts
2122
PowerShellPlatform: PowerShell Core
2223

2324
trigger: none

‎.azure-pipelines/windows-powershell.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ variables:
1010
BuildTimeoutInMinutes: 120
1111
AnalysisTimeoutInMinutes: 120
1212
EnableTestCoverage: true
13+
TestCoverageLocation: $(Build.SourcesDirectory)/artifacts
1314
PowerShellPlatform: Windows PowerShell
1415

1516
trigger: none

‎src/Accounts/Authentication/Config/ConfigInitializer.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,12 @@ private void RegisterConfigs(IConfigManager configManager)
197197
false,
198198
ConfigKeys.EnableTestCoverage,
199199
new[] { AppliesTo.Az }));
200+
configManager.RegisterConfig(new SimpleTypedConfig<string>(
201+
ConfigKeys.TestCoverageLocation,
202+
"Only takes effect when EnableTestCoverage equals to TRUE. Use this config to redirect the test coverage data location.",
203+
string.Empty,
204+
ConfigKeys.TestCoverageLocation,
205+
new[] { AppliesTo.Az }));
200206
#endif
201207

202208
configManager.RegisterConfig(new EnableInterceptSurveyConfig());

‎src/Accounts/Authentication/TestCoverage.cs‎

Lines changed: 32 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,13 @@
1212
// limitations under the License.
1313
// ----------------------------------------------------------------------------------
1414

15+
using Microsoft.Azure.Commands.Common.Authentication.Properties;
16+
using Microsoft.Azure.Commands.Shared.Config;
17+
using Microsoft.Azure.PowerShell.Common.Config;
1518
using Microsoft.WindowsAzure.Commands.Common;
1619
using System;
1720
using System.Collections.Generic;
1821
using System.IO;
19-
using System.Runtime.InteropServices;
2022
using System.Text;
2123
using System.Text.RegularExpressions;
2224
using System.Threading;
@@ -30,6 +32,8 @@ public class TestCoverage : ITestCoverage
3032
private const string CsvHeaderParameters = "Parameters";
3133
private const string CsvHeaderSourceScript = "SourceScript";
3234
private const string CsvHeaderScriptLineNumber = "LineNumber";
35+
private const string CsvHeaderStartDateTime = "StartDateTime";
36+
private const string CsvHeaderEndDateTime = "EndDateTime";
3337
private const string CsvHeaderIsSuccess = "IsSuccess";
3438
private const string Delimiter = ",";
3539

@@ -41,36 +45,32 @@ public class TestCoverage : ITestCoverage
4145
"AzureRM.Storage.ps1"
4246
};
4347

44-
45-
private static readonly string s_testCoverageRootPath;
48+
private static readonly string s_testCoveragePath;
4649

4750
private static readonly ReaderWriterLockSlim s_lock = new ReaderWriterLockSlim();
4851

4952
static TestCoverage()
5053
{
51-
52-
var repoRootPath = ProbeRepoDirectory();
53-
if (!string.IsNullOrEmpty(repoRootPath))
54+
string testCoverageRootPath = null;
55+
if (AzureSession.Instance.TryGetComponent<IConfigManager>(nameof(IConfigManager), out var configManager))
5456
{
55-
s_testCoverageRootPath = Path.Combine(repoRootPath, "artifacts", "TestCoverageAnalysis", "Raw");
56-
DirectoryInfo rawDir = new DirectoryInfo(s_testCoverageRootPath);
57-
if (!rawDir.Exists)
58-
{
59-
Directory.CreateDirectory(s_testCoverageRootPath);
60-
}
57+
testCoverageRootPath = configManager.GetConfigValue<string>(ConfigKeys.TestCoverageLocation);
58+
}
59+
if (string.IsNullOrEmpty(testCoverageRootPath))
60+
{
61+
testCoverageRootPath = Path.Combine(
62+
Environment.GetFolderPath(Environment.SpecialFolder.UserProfile),
63+
Resources.AzureDirectoryName);
6164
}
62-
}
6365

64-
private static string ProbeRepoDirectory()
65-
{
66-
string directoryPath = "..";
67-
while (Directory.Exists(directoryPath) && (!Directory.Exists(Path.Combine(directoryPath, "src")) || !Directory.Exists(Path.Combine(directoryPath, "artifacts"))))
66+
s_testCoveragePath = Path.Combine(testCoverageRootPath, "TestCoverageAnalysis", "Raw");
67+
DirectoryInfo rawDir = new DirectoryInfo(s_testCoveragePath);
68+
if (!rawDir.Exists)
6869
{
69-
directoryPath = Path.Combine(directoryPath, "..");
70+
Directory.CreateDirectory(s_testCoveragePath);
7071
}
7172

72-
string result = Directory.Exists(directoryPath) ? Path.GetFullPath(directoryPath) : null;
73-
return result;
73+
Console.WriteLine($"Test coverage data location: ${s_testCoveragePath}");
7474
}
7575

7676
private string GenerateCsvHeader()
@@ -81,19 +81,23 @@ private string GenerateCsvHeader()
8181
.Append(CsvHeaderParameters).Append(Delimiter)
8282
.Append(CsvHeaderSourceScript).Append(Delimiter)
8383
.Append(CsvHeaderScriptLineNumber).Append(Delimiter)
84+
.Append(CsvHeaderStartDateTime).Append(Delimiter)
85+
.Append(CsvHeaderEndDateTime).Append(Delimiter)
8486
.Append(CsvHeaderIsSuccess);
8587

8688
return headerBuilder.ToString();
8789
}
8890

89-
private string GenerateCsvItem(string commandName, string parameterSetName, string parameters, string sourceScript, int scriptLineNumber, bool isSuccess)
91+
private string GenerateCsvItem(string commandName, string parameterSetName, string parameters, string sourceScript, int scriptLineNumber, string startDateTime, string endDateTime, bool isSuccess)
9092
{
9193
StringBuilder itemBuilder = new StringBuilder();
9294
itemBuilder.Append(commandName).Append(Delimiter)
9395
.Append(parameterSetName).Append(Delimiter)
9496
.Append(parameters).Append(Delimiter)
9597
.Append(sourceScript).Append(Delimiter)
9698
.Append(scriptLineNumber).Append(Delimiter)
99+
.Append(startDateTime).Append(Delimiter)
100+
.Append(endDateTime).Append(Delimiter)
97101
.Append(isSuccess.ToString().ToLowerInvariant());
98102

99103
return itemBuilder.ToString();
@@ -104,18 +108,19 @@ public void LogRawData(AzurePSQoSEvent qos)
104108
#if DEBUG || TESTCOVERAGE
105109
string moduleName = qos.ModuleName;
106110
string commandName = qos.CommandName;
107-
string sourceScript = qos.SourceScript;
111+
string sourceScriptPath = qos.SourceScript;
112+
string sourceScriptName = Path.GetFileName(sourceScriptPath);
108113

109-
if (string.IsNullOrEmpty(moduleName) || string.IsNullOrEmpty(commandName) || ExcludedSource.Contains(sourceScript))
114+
if (string.IsNullOrEmpty(moduleName) || string.IsNullOrEmpty(commandName) || string.IsNullOrEmpty(sourceScriptName) || ExcludedSource.Contains(sourceScriptName))
110115
return;
111116

112-
var pattern = @"\\(?:artifacts\\Debug|src)\\(?:Az\.)?(?<ModuleName>[a-zA-Z]+)\\";
113-
var match = Regex.Match(sourceScript, pattern, RegexOptions.IgnoreCase);
117+
var pattern = @"[\\|\/](?:artifacts[\\|\/]Debug|src)[\\|\/](?:Az\.)?(?<ModuleName>[a-zA-Z]+)[\\|\/]";
118+
var match = Regex.Match(sourceScriptPath, pattern, RegexOptions.IgnoreCase);
114119
var testingModuleName = $"Az.{match.Groups["ModuleName"].Value}";
115120
if (string.Compare(testingModuleName, moduleName, true) != 0)
116121
return;
117122

118-
var csvFilePath = Path.Combine(s_testCoverageRootPath, $"{moduleName}.csv");
123+
var csvFilePath = Path.Combine(s_testCoveragePath, $"{moduleName}.csv");
119124
StringBuilder csvData = new StringBuilder();
120125

121126
s_lock.EnterWriteLock();
@@ -128,7 +133,7 @@ public void LogRawData(AzurePSQoSEvent qos)
128133
}
129134

130135
csvData.AppendLine();
131-
var csvItem = GenerateCsvItem(commandName, qos.ParameterSetName, qos.Parameters, Path.GetFileName(sourceScript), qos.ScriptLineNumber, qos.IsSuccess);
136+
var csvItem = GenerateCsvItem(commandName, qos.ParameterSetName, qos.Parameters, sourceScriptName, qos.ScriptLineNumber, qos.StartTime.UtcDateTime.ToString("yyyy-MM-ddTHH:mm:ss"), qos.EndTime.UtcDateTime.ToString("yyyy-MM-ddTHH:mm:ss"), qos.IsSuccess);
132137
csvData.Append(csvItem);
133138

134139
File.AppendAllText(csvFilePath, csvData.ToString());

‎src/shared/ConfigKeys.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,6 @@ internal static class ConfigKeys
2929
public const string DefaultSubscriptionForLogin = "DefaultSubscriptionForLogin";
3030
public const string EnableDataCollection = "EnableDataCollection";
3131
public const string EnableTestCoverage = "EnableTestCoverage";
32+
public const string TestCoverageLocation = "TestCoverageLocation";
3233
}
3334
}

tools/TestCoverageAnalysis/AnalyzeTestCoverage.ps1 renamed to tools/TestFx/Coverage/AnalyzeTestCoverage.ps1

Lines changed: 69 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -47,35 +47,33 @@ $excludedModules = @(
4747
"Az.Elastic",
4848
"Az.HanaOnAzure",
4949
"Az.HealthBot",
50-
"Az.ImportExport",
5150
"Az.LabServices",
5251
"Az.Logz",
5352
"Az.ManagedServices",
5453
"Az.Maps",
5554
"Az.MariaDb",
5655
"Az.Marketplace",
5756
"Az.MonitoringSolutions",
58-
"Az.MySql",
5957
"Az.Portal",
6058
"Az.PostgreSql",
6159
"Az.ProviderHub",
6260
"Az.Purview",
6361
"Az.Quota",
6462
"Az.ResourceGraph",
6563
"Az.ResourceMover",
66-
"Az.SignalR",
6764
"Az.StreamAnalytics",
6865
"Az.Synapse",
6966
"Az.TimeSeriesInsights",
7067
"Az.Websites",
7168
"Az.WindowsIotServices"
7269
)
7370

74-
if (-not (Test-Path -LiteralPath $testCoverageResultsDirectory -PathType Container))
75-
{
71+
if (!(Test-Path -LiteralPath $testCoverageResultsDirectory -PathType Container)) {
7672
New-Item -Path $testCoverageRootDirectory -Name "Results" -ItemType Directory
7773
}
78-
Get-ChildItem -LiteralPath $testCoverageResultsDirectory -Filter "*.txt" | Remove-Item -Force
74+
else {
75+
Get-ChildItem -LiteralPath $testCoverageResultsDirectory -Filter "*.txt" | Remove-Item -Force
76+
}
7977

8078
$accountModuleName = "Az.Accounts"
8179
$accountModuleFullPath = Join-Path -Path $debugDirectory -ChildPath $accountModuleName | Join-Path -ChildPath "$accountModuleName.psd1"
@@ -89,22 +87,23 @@ $AzPSReportContent = [System.Text.StringBuilder]::new()
8987
[void]$AzPSReportContent.AppendLine()
9088

9189
$allModules = Get-ChildItem -LiteralPath $debugDirectory -Filter "Az.*" -Directory -Name
92-
foreach ($moduleName in $allModules)
93-
{
94-
if ($moduleName -in $excludedModules)
95-
{
90+
foreach ($moduleName in $allModules) {
91+
Write-Host "##[group]Start to analyze test coverage for the module '$moduleName'" -ForegroundColor Cyan
92+
93+
if ($moduleName -in $excludedModules) {
94+
Write-Warning "The module '$moduleName' has been excluded."
95+
Write-Host "##[endgroup]" -ForegroundColor Cyan
9696
continue
9797
}
9898

99-
Write-Host "##[group]Calculating test coverage for module $moduleName"
100-
Write-Host "##[section]Start analyzing module $moduleName" -ForegroundColor Green
99+
$hasRawData = $true
101100

102101
$moduleCsvFullPath = Join-Path -Path $testCoverageRawDirectory -ChildPath "$moduleName.csv"
103-
if (-not (Test-Path $moduleCsvFullPath))
104-
{
105-
[void]$AzPSReportContent.AppendLine(" Module name: $moduleName has no test raw data found!")
102+
if (!(Test-Path $moduleCsvFullPath)) {
103+
Write-Warning "The module '$moduleName' has no test raw data generated."
104+
[void]$AzPSReportContent.AppendLine(" The module '$moduleName' has no test raw data file found!")
106105
[void]$AzPSReportContent.AppendLine()
107-
continue
106+
$hasRawData = $false
108107
}
109108

110109
[void]$AzPSReportContent.AppendLine(" Module name: $moduleName")
@@ -113,28 +112,17 @@ foreach ($moduleName in $allModules)
113112
[void]$AzPSModuleReportContent.AppendLine("Test coverage report for module $moduleName :")
114113
[void]$AzPSModuleReportContent.AppendLine()
115114

116-
if ($moduleName -ne $accountModuleName)
117-
{
115+
if ($moduleName -ne $accountModuleName) {
118116
$moduleFullPath = Join-Path -Path $debugDirectory -ChildPath $moduleName | Join-Path -ChildPath "$moduleName.psd1"
119117
Import-Module $moduleFullPath
120118
}
121119

122-
Write-Host "##[section]Start calculating cmdlet ..." -ForegroundColor Green
123-
124120
$module = Get-Module -Name $moduleName
125121
$moduleCmdlets = $module.ExportedCmdlets.Keys + $module.ExportedFunctions.Keys
126122

127123
$totalCmdletsCount = $moduleCmdlets.Count
128124
$overallCmdletsCount += $totalCmdletsCount
129125

130-
$rawCsv = Import-Csv -LiteralPath $moduleCsvFullPath | Select-Object * -Unique
131-
132-
$csvGroupByCmdlet = $rawCsv | Select-Object -ExpandProperty CommandName -Unique | Sort-Object CommandName
133-
$totalExecutedCmdletsCount = $csvGroupByCmdlet.Count
134-
$overallExecutedCmdletsCount += $totalExecutedCmdletsCount
135-
136-
Write-Host "##[section]Start calculating parameter set ..." -ForegroundColor Green
137-
138126
$totalParameterSetsCount = 0
139127
$totalParametersCount = 0
140128
$moduleCmdlets | ForEach-Object {
@@ -143,53 +131,75 @@ foreach ($moduleName in $allModules)
143131

144132
$cmdletParams = $cmdlet.Parameters
145133
$cmdletParams.Keys | ForEach-Object {
146-
if ($_ -notin $psCommonParameters)
147-
{
134+
if ($_ -notin $psCommonParameters) {
148135
$totalParametersCount += $cmdletParams[$_].ParameterSets.Count
149136
}
150137
}
151138
}
152139

153-
$csvGroupByParameterSet = $rawCsv | Select-Object CommandName, ParameterSetName -Unique | Sort-Object CommandName, ParameterSetName | Group-Object CommandName
154-
$totalExecutedParameterSetsCount = ($csvGroupByParameterSet | Measure-Object -Property Count -Sum).Sum
140+
if ($hasRawData) {
141+
$rawCsv = Import-Csv -LiteralPath $moduleCsvFullPath | Select-Object * -Unique
142+
143+
Write-Host "##[section]Start to calculate cmdlets." -ForegroundColor Green
144+
145+
$csvGroupByCmdlet = $rawCsv | Select-Object -ExpandProperty CommandName -Unique | Sort-Object CommandName
146+
$totalExecutedCmdletsCount = $csvGroupByCmdlet.Count
147+
$overallExecutedCmdletsCount += $totalExecutedCmdletsCount
148+
149+
Write-Host "##[section]Finished calculating cmdlets." -ForegroundColor Green
150+
151+
Write-Host "##[section]Start to calculate parameter sets." -ForegroundColor Green
152+
153+
$csvGroupByParameterSet = $rawCsv | Select-Object CommandName, ParameterSetName -Unique | Sort-Object CommandName, ParameterSetName | Group-Object CommandName
154+
$totalExecutedParameterSetsCount = ($csvGroupByParameterSet | Measure-Object -Property Count -Sum).Sum
155155

156-
[void]$AzPSModuleReportContent.AppendLine("Following cmdlets were executed :")
157-
$csvGroupByParameterSet | ForEach-Object {
158-
$_.Group | ForEach-Object {
159-
[void]$AzPSModuleReportContent.AppendLine(" --> Cmdlet $($_.CommandName) with parameter set $($_.ParameterSetName) was tested")
156+
[void]$AzPSModuleReportContent.AppendLine("Following cmdlets were executed :")
157+
$csvGroupByParameterSet | ForEach-Object {
158+
$_.Group | ForEach-Object {
159+
[void]$AzPSModuleReportContent.AppendLine(" --> Cmdlet $($_.CommandName) with parameter set $($_.ParameterSetName) was tested")
160+
}
160161
}
161-
}
162162

163-
[void]$AzPSModuleReportContent.AppendLine()
163+
[void]$AzPSModuleReportContent.AppendLine()
164164

165-
[void]$AzPSModuleReportContent.AppendLine("Following cmdlets were not executed :")
166-
$moduleCmdlets | Where-Object { $_ -notin $csvGroupByCmdlet } | ForEach-Object {
167-
[void]$AzPSModuleReportContent.AppendLine(" --> $_")
168-
}
165+
[void]$AzPSModuleReportContent.AppendLine("Following cmdlets were not executed :")
166+
$moduleCmdlets | Where-Object { $_ -notin $csvGroupByCmdlet } | ForEach-Object {
167+
[void]$AzPSModuleReportContent.AppendLine(" --> $_")
168+
}
169+
170+
Write-Host "##[section]Finished calculating parameter sets." -ForegroundColor Green
169171

170-
Write-Host "##[section]Start calculating parameter ..." -ForegroundColor Green
172+
Write-Host "##[section]Start to calculate parameters." -ForegroundColor Green
171173

172-
$csvGroupByParameter = $rawCsv | Select-Object CommandName, ParameterSetName, Parameters -Unique | Sort-Object CommandName, ParameterSetName, Parameters | Group-Object CommandName, ParameterSetName
173-
$totalExecutedParametersCount = 0
174-
$csvGroupByParameter | ForEach-Object {
175-
$executedParams = @()
176-
$_.Group | ForEach-Object {
177-
$executedParams += $_.Parameters -split "\*\*\*\s*"
174+
$csvGroupByParameter = $rawCsv | Select-Object CommandName, ParameterSetName, Parameters -Unique | Sort-Object CommandName, ParameterSetName, Parameters | Group-Object CommandName, ParameterSetName
175+
$totalExecutedParametersCount = 0
176+
$csvGroupByParameter | ForEach-Object {
177+
$executedParams = @()
178+
$_.Group | ForEach-Object {
179+
$executedParams += $_.Parameters -split "\*\*\*\s*"
180+
}
181+
$totalExecutedParametersCount += ($executedParams | Where-Object { $_ -and $_ -notin $psCommonParameters } | Select-Object -Unique).Count
178182
}
179-
$totalExecutedParametersCount += ($executedParams | Where-Object { $_ -and $_ -notin $psCommonParameters } | Select-Object -Unique).Count
180-
}
181183

182-
$AzPSModuleReportContent.ToString() | Out-File -LiteralPath ([System.IO.Path]::Combine($testCoverageResultsDirectory, "$moduleName.txt")) -NoNewLine
184+
Write-Host "##[section]Finished calculating parameters." -ForegroundColor Green
185+
186+
$AzPSModuleReportContent.ToString() | Out-File -LiteralPath ([System.IO.Path]::Combine($testCoverageResultsDirectory, "$moduleName.txt")) -NoNewline
183187

184-
[void]$AzPSReportContent.AppendLine(" --> By cmdlet: $(($totalExecutedCmdletsCount / $totalCmdletsCount).ToString("P0"))")
185-
[void]$AzPSReportContent.AppendLine(" --> By parameter set: $(($totalExecutedParameterSetsCount / $totalParameterSetsCount).ToString("P0"))")
186-
[void]$AzPSReportContent.AppendLine(" --> By parameter: $(($totalExecutedParametersCount / $totalParametersCount).ToString("P0"))")
187-
[void]$AzPSReportContent.AppendLine()
188+
[void]$AzPSReportContent.AppendLine(" --> By cmdlet: $(($totalExecutedCmdletsCount / $totalCmdletsCount).ToString("P0"))")
189+
[void]$AzPSReportContent.AppendLine(" --> By parameter set: $(($totalExecutedParameterSetsCount / $totalParameterSetsCount).ToString("P0"))")
190+
[void]$AzPSReportContent.AppendLine(" --> By parameter: $(($totalExecutedParametersCount / $totalParametersCount).ToString("P0"))")
191+
[void]$AzPSReportContent.AppendLine()
192+
}
188193

189-
Write-Host "##[section]Finished analyzing module $moduleName" -ForegroundColor Green
190-
Write-Host "##[endgroup]"
194+
Write-Host "##[section]Successfully analyzed module '$moduleName'." -ForegroundColor Green
195+
Write-Host "##[endgroup]" -ForegroundColor Cyan
191196
Write-Host
192197
}
193198

199+
Write-Host "Total tested cmdlets count: $overallExecutedCmdletsCount" -ForegroundColor Cyan
200+
Write-Host "Total cmdlets count: $overallCmdletsCount" -ForegroundColor Cyan
201+
202+
[void]$AzPSReportContent.AppendLine("Total tested cmdlets count: $overallExecutedCmdletsCount")
203+
[void]$AzPSReportContent.AppendLine("Total cmdlets count: $overallCmdletsCount")
194204
[void]$AzPSReportContent.AppendLine("Total Azure PowerShell cmdlets coverage: $(($overallExecutedCmdletsCount / $overallCmdletsCount).ToString("P0"))")
195-
$AzPSReportContent.ToString() | Out-File -LiteralPath ([System.IO.Path]::Combine($testCoverageResultsDirectory, "Azure PowerShell Test Coverage Report.txt")) -NoNewLine
205+
$AzPSReportContent.ToString() | Out-File -LiteralPath ([System.IO.Path]::Combine($testCoverageResultsDirectory, "Azure PowerShell Test Coverage Report.txt")) -NoNewline

0 commit comments

Comments
 (0)