Sync eng/common directory with azure-sdk-tools for PR 1429 (Azure#16826)

* Add job matrix generation scripts

* Add working job matrix example pipeline and common matrix generation pipeline.

* Update job matrix tests path

* Parameterize matrix generation job path

* Add global variable to override nuget security checks to sample matrix pipeline

* Update readme matrix pipeline example to match sample file

Co-authored-by: Ben Broderick Phillips <bebroder@microsoft.com>

Author: azure-sdk

eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml

@@ -0,0 +1,73 @@
+parameters:
+- name: AdditionalParameters
+  type: object
+- name: CloudConfig
+  type: object
+  default: {}
+- name: MatrixConfigs
+  type: object
+  default: []
+- name: MatrixFilters
+  type: object
+  default: []
+- name: JobTemplatePath
+  type: string
+
+jobs:
+- job: generate_matrix
+  variables:
+    displayNameFilter: $[ coalesce(variables.jobMatrixFilter, '.*') ]
+  pool:
+    name: Azure Pipelines
+    vmImage: ubuntu-18.04
+  displayName: Generate Job Matrix
+  steps:
+    - ${{ each config in parameters.MatrixConfigs }}:
+      - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+        - task: Powershell@2
+          inputs:
+            pwsh: true
+            filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+            arguments: >
+              -ConfigPath ${{ config.Path }}
+              -Selection ${{ config.Selection }}
+              -DisplayNameFilter "$(displayNameFilter)"
+              -Filters "${{ join('","', parameters.MatrixFilters) }}","container=^$","SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}"
+              -NonSparseParameters "${{ join('","', config.NonSparseParameters) }}"
+          displayName: Generate VM Job Matrix ${{ config.Name }}
+          name: generate_vm_job_matrix_${{ config.Name }}
+
+      - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+        - task: Powershell@2
+          inputs:
+            pwsh: true
+            filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+            arguments: >
+              -ConfigPath ${{ config.Path }}
+              -Selection ${{ config.Selection }}
+              -DisplayNameFilter "$(displayNameFilter)"
+              -Filters "${{ join('","', parameters.MatrixFilters) }}", "container=.*", "SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}"
+              -NonSparseParameters "${{ join('","', config.NonSparseParameters) }}"
+          displayName: Generate Container Job Matrix
+          name: generate_container_job_matrix_${{ config.Name }}
+
+- ${{ each config in parameters.MatrixConfigs }}:
+  - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+    - template: ${{ parameters.JobTemplatePath }}
+      parameters:
+        UsePlatformContainer: false
+        Matrix: dependencies.generate_matrix.outputs['generate_vm_job_matrix_${{ config.Name }}.matrix']
+        DependsOn: generate_matrix
+        CloudConfig: ${{ parameters.CloudConfig }}
+        ${{ each param in parameters.AdditionalParameters }}:
+          ${{ param.key }}: ${{ param.value }}
+
+  - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+    - template: ${{ parameters.JobTemplatePath }}
+      parameters:
+        UsePlatformContainer: true
+        Matrix: dependencies.generate_matrix.outputs['generate_container_job_matrix_${{ config.Name }}.matrix']
+        DependsOn: generate_matrix
+        CloudConfig: ${{ parameters.CloudConfig }}
+        ${{ each param in parameters.AdditionalParameters }}:
+          ${{ param.key }}: ${{ param.value }}

eng/common/scripts/job-matrix/Create-JobMatrix.ps1

@@ -0,0 +1,35 @@
+<#
+    .SYNOPSIS
+        Generates a JSON object representing an Azure Pipelines Job Matrix.
+        See https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#parallelexec
+
+    .EXAMPLE
+    ./eng/common/scripts/Create-JobMatrix $context
+#>
+
+[CmdletBinding()]
+param (
+    [Parameter(Mandatory=$True)][string] $ConfigPath,
+    [Parameter(Mandatory=$True)][string] $Selection,
+    [Parameter(Mandatory=$False)][string] $DisplayNameFilter,
+    [Parameter(Mandatory=$False)][array] $Filters,
+    [Parameter(Mandatory=$False)][array] $NonSparseParameters
+)
+
+. $PSScriptRoot/job-matrix-functions.ps1
+
+$config = GetMatrixConfigFromJson (Get-Content $ConfigPath)
+# Strip empty string filters in order to be able to use azure pipelines yaml join()
+$Filters = $Filters | Where-Object { $_ }
+
+[array]$matrix = GenerateMatrix `
+    -config $config `
+    -selectFromMatrixType $Selection `
+    -displayNameFilter $DisplayNameFilter `
+    -filters $Filters `
+    -nonSparseParameters $NonSparseParameters
+
+$serialized = SerializePipelineMatrix $matrix
+
+Write-Output $serialized.pretty
+Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)"