Sync eng/common directory with azure-sdk-tools for PR 3790 (Azure#31582)

azure-sdk · ckairen · web-flow · commit e54541bfe754 · 2022-10-20T16:30:06.000-07:00
* Added yaml support for job matrix creation

* autogen scenario matrix for stress test

* Temporary Working State

* update to default sparse

* pr comments and some error handling

* custom matrixfilename and ordering of generatedValues.yaml

* common module import

* JobMatrix write host

Co-authored-by: Albert Cheng &lt;albertcheng@microsoft.com&gt;
diff --git a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 b/eng/common/scripts/Helpers/PSModule-Helpers.ps1
@@ -22,9 +22,9 @@ function Update-PSModulePathForCI()
   $modulePaths = $modulePaths.Where({ !$_.StartsWith($hostedAgentModulePath) })
 
   # Add any "az_" paths from the agent which is the lastest set of azure modules
-  $AzModuleCachPath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator
-  if ($AzModuleCachPath -and $env.PSModulePath -notcontains $AzModuleCachPath) {
-    $modulePaths += $AzModuleCachPath
+  $AzModuleCachePath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator
+  if ($AzModuleCachePath -and $env:PSModulePath -notcontains $AzModuleCachePath) {
+    $modulePaths += $AzModuleCachePath
   }
 
   $env:PSModulePath = $modulePaths -join $moduleSeperator
diff --git a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
@@ -23,7 +23,7 @@ if (!(Test-Path $ConfigPath)) {
     Write-Error "ConfigPath '$ConfigPath' does not exist."
     exit 1
 }
-$config = GetMatrixConfigFromJson (Get-Content $ConfigPath)
+$config = GetMatrixConfigFromFile (Get-Content $ConfigPath -Raw)
 # Strip empty string filters in order to be able to use azure pipelines yaml join()
 $Filters = $Filters | Where-Object { $_ }
 
@@ -38,4 +38,7 @@ $Filters = $Filters | Where-Object { $_ }
 $serialized = SerializePipelineMatrix $matrix
 
 Write-Output $serialized.pretty
-Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)"
+
+if ($null -ne $env:SYSTEM_TEAMPROJECTID) {
+    Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)"
+}
diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1
@@ -84,6 +84,7 @@ class MatrixParameter {
     }
 }
 
+. (Join-Path $PSScriptRoot "../Helpers" PSModule-Helpers.ps1)
 $IMPORT_KEYWORD = '$IMPORT'
 
 function GenerateMatrix(
@@ -146,7 +147,7 @@ function ProcessNonSparseParameters(
 
 function FilterMatrixDisplayName([array]$matrix, [string]$filter) {
     return $matrix | Where-Object { $_ } | ForEach-Object {
-        if ($_.Name -match $filter) {
+        if ($_.ContainsKey("Name") -and $_.Name -match $filter) {
             return $_
         }
     }
@@ -168,7 +169,7 @@ function MatchesFilters([hashtable]$entry, [array]$filters) {
         # Default all regex checks to go against empty string when keys are missing.
         # This simplifies the filter syntax/interface to be regex only.
         $value = ""
-        if ($null -ne $entry -and $entry.parameters.Contains($key)) {
+        if ($null -ne $entry -and $entry.ContainsKey("parameters") -and $entry.parameters.Contains($key)) {
             $value = $entry.parameters[$key]
         }
         if ($value -notmatch $regex) {
@@ -190,11 +191,34 @@ function ParseFilter([string]$filter) {
     }
 }
 
-# Importing the JSON as PSCustomObject preserves key ordering,
-# whereas ConvertFrom-Json -AsHashtable does not
+function GetMatrixConfigFromFile([String] $config)
+{
+    [MatrixConfig]$config = try{
+        GetMatrixConfigFromJson $config
+    } catch {
+        GetMatrixConfigFromYaml $config
+    }
+    return $config
+}
+
+function GetMatrixConfigFromYaml([String] $yamlConfig)
+{
+    Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
+    # ConvertTo then from json is to make sure the nested values are in PSCustomObject
+    [MatrixConfig]$config = ConvertFrom-Yaml $yamlConfig -Ordered | ConvertTo-Json -Depth 100 | ConvertFrom-Json
+    return GetMatrixConfig $config
+}
+
 function GetMatrixConfigFromJson([String]$jsonConfig)
 {
     [MatrixConfig]$config = $jsonConfig | ConvertFrom-Json
+    return GetMatrixConfig $config
+}
+
+# Importing the JSON as PSCustomObject preserves key ordering,
+# whereas ConvertFrom-Json -AsHashtable does not
+function GetMatrixConfig([MatrixConfig]$config)
+{
     $config.matrixParameters = @()
     $config.displayNamesLookup = @{}
     $include = [MatrixParameter[]]@()
@@ -359,7 +383,7 @@ function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$n
         Write-Error "`$IMPORT path '$importPath' does not exist."
         exit 1
     }
-    $importedMatrixConfig = GetMatrixConfigFromJson (Get-Content $importPath)
+    $importedMatrixConfig = GetMatrixConfigFromFile (Get-Content -Raw $importPath)
     $importedMatrix = GenerateMatrix `
                         -config $importedMatrixConfig `
                         -selectFromMatrixType $selection `
diff --git a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1
@@ -24,7 +24,15 @@ param(
     [string]$Namespace,
 
     # Override remote stress-test-addons with local on-disk addons for development
-    [System.IO.FileInfo]$LocalAddonsPath
+    [System.IO.FileInfo]$LocalAddonsPath,
+
+    # Matrix generation parameters
+    [Parameter(Mandatory=$False)][string]$MatrixFileName,
+    [Parameter(Mandatory=$False)][string]$MatrixSelection,
+    [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter,
+    [Parameter(Mandatory=$False)][array]$MatrixFilters,
+    [Parameter(Mandatory=$False)][array]$MatrixReplace,
+    [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters
 )
 
 . $PSScriptRoot/stress-test-deployment-lib.ps1
diff --git a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1
@@ -12,20 +12,42 @@ class StressTestPackageInfo {
     [string]$Deployer
 }
 
+. $PSScriptRoot/../job-matrix/job-matrix-functions.ps1
+. $PSScriptRoot/generate-scenario-matrix.ps1
+
 function FindStressPackages(
     [string]$directory,
     [hashtable]$filters = @{},
     [switch]$CI,
-    [string]$namespaceOverride
+    [string]$namespaceOverride,
+    [string]$MatrixSelection,
+    [Parameter(Mandatory=$False)][string]$MatrixFileName,
+    [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter,
+    [Parameter(Mandatory=$False)][array]$MatrixFilters,
+    [Parameter(Mandatory=$False)][array]$MatrixReplace,
+    [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters
 ) {
     # Bare minimum filter for stress tests
     $filters['stressTest'] = 'true'
-
     $packages = @()
     $chartFiles = Get-ChildItem -Recurse -Filter 'Chart.yaml' $directory 
+    if (!$MatrixFileName) {
+        $MatrixFileName = '/scenarios-matrix.yaml'
+    }
     foreach ($chartFile in $chartFiles) {
         $chart = ParseChart $chartFile
         if (matchesAnnotations $chart $filters) {
+            $matrixFilePath = (Join-Path $chartFile.Directory.FullName $MatrixFileName)
+            if (Test-Path $matrixFilePath) {
+                GenerateScenarioMatrix `
+                    -matrixFilePath $matrixFilePath `
+                    -Selection $MatrixSelection `
+                    -DisplayNameFilter $MatrixDisplayNameFilter `
+                    -Filters $MatrixFilters `
+                    -Replace $MatrixReplace `
+                    -NonSparseParameters $MatrixNonSparseParameters
+            }
+
             $packages += NewStressTestPackageInfo `
                             -chart $chart `
                             -chartFile $chartFile `
@@ -80,8 +102,8 @@ function NewStressTestPackageInfo(
         Namespace = $namespace.ToLower()
         Directory = $chartFile.DirectoryName
         ReleaseName = $chart.name
-        Dockerfile = $chart.annotations.dockerfile
-        DockerBuildDir = $chart.annotations.dockerbuilddir
+        Dockerfile = "dockerfile" -in $chart.annotations.keys ? $chart.annotations.dockerfile : $null
+        DockerBuildDir = "dockerbuilddir" -in $chart.annotations.keys ? $chart.annotations.dockerbuilddir : $null
     }
 }
 
diff --git a/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 b/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1
@@ -0,0 +1,86 @@
+param(
+    [string]$matrixFilePath,
+    [string]$Selection,
+    [Parameter(Mandatory=$False)][string]$DisplayNameFilter,
+    [Parameter(Mandatory=$False)][array]$Filters,
+    [Parameter(Mandatory=$False)][array]$Replace,
+    [Parameter(Mandatory=$False)][array]$NonSparseParameters
+)
+
+function GenerateScenarioMatrix(
+    [string]$matrixFilePath,
+    [string]$Selection,
+    [Parameter(Mandatory=$False)][string]$DisplayNameFilter,
+    [Parameter(Mandatory=$False)][array]$Filters,
+    [Parameter(Mandatory=$False)][array]$Replace,
+    [Parameter(Mandatory=$False)][array]$NonSparseParameters
+) {
+    $yamlConfig = Get-Content $matrixFilePath -Raw
+
+    $prettyMatrix = &"$PSScriptRoot/../job-matrix/Create-JobMatrix.ps1" `
+        -ConfigPath $matrixFilePath `
+        -Selection $Selection `
+        -DisplayNameFilter $DisplayNameFilter `
+        -Filters $Filters `
+        -Replace $Replace `
+        -NonSparseParameters $NonSparseParameters
+    Write-Host $prettyMatrix
+    $prettyMatrix = $prettyMatrix | ConvertFrom-Json
+
+    $scenariosMatrix = @()
+    foreach($permutation in $prettyMatrix.psobject.properties) {
+        $entry = @{}
+        $entry.Name = $permutation.Name -replace '_', '-'
+        $entry.Scenario = $entry.Name
+        $entry.Remove("Name")
+        foreach ($param in $permutation.value.psobject.properties) {
+            $entry.add($param.Name, $param.value)
+        }
+        $scenariosMatrix += $entry
+    }
+
+    $valuesYaml = Get-Content -Raw (Join-Path (Split-Path $matrixFilePath) 'values.yaml')
+    $values = $valuesYaml | ConvertFrom-Yaml -Ordered
+    if (!$values) {$values = @{}}
+
+    if ($values.ContainsKey('Scenarios')) {
+        throw "Please use matrix generation for stress test scenarios."
+    }
+
+    $values.scenarios = $scenariosMatrix
+    $values | ConvertTo-Yaml | Out-File -FilePath (Join-Path $matrixFilePath '../generatedValues.yaml')
+}
+
+function NewStressTestPackageInfo(
+    [hashtable]$chart,
+    [System.IO.FileInfo]$chartFile,
+    [switch]$CI,
+    [object]$namespaceOverride
+) {
+    $namespace = if ($namespaceOverride) {
+        $namespaceOverride
+    } elseif ($CI) {
+        $chart.annotations.namespace
+    } else {
+        GetUsername
+    }
+
+    return [StressTestPackageInfo]@{
+        Namespace = $namespace.ToLower()
+        Directory = $chartFile.DirectoryName
+        ReleaseName = $chart.name
+        Dockerfile = $chart.annotations.dockerfile
+        DockerBuildDir = $chart.annotations.dockerbuilddir
+    }
+}
+
+# Don't call functions when the script is being dot sourced
+if ($MyInvocation.InvocationName -ne ".") {
+    GenerateScenarioMatrix `
+        -matrixFilePath $matrixFilePath `
+        -Selection $Selection `
+        -DisplayNameFilter $DisplayNameFilter `
+        -Filters $Filters `
+        -Replace $Replace `
+        -NonSparseParameters $NonSparseParameters
+}
diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1
