Update live tests to use generated matrix configuration (Azure#18131)

* Update live tests to use generated matrix configuration. Update generation features.

* Simplify matrix config filter code

* Use sparse matrix for identity live tests

* Invert base matrix selection. Fix matrix object field dimension bug.

* Simplify matrix filter regex filter syntax

* Fix windows pool name in platform matrix

* Change live test platform matrix configuration to sparse

* Add UseProjectRef on net5.0 to live test matrix

Author: benbp

eng/pipelines/templates/jobs/archetype-sdk-tests-generate.yml

@@ -0,0 +1,62 @@
+parameters:
+- name: AdditionalParameters
+  type: object
+- name: CloudConfig
+  type: object
+  default: {}
+- name: MatrixConfigs
+  type: object
+  default: []
+
+jobs:
+- job: generate_matrix
+  variables:
+    displayNameFilter: $[ coalesce(variables.jobMatrixFilter, '.*') ]
+  pool:
+    name: azsdk-pool-mms-ubuntu-1804-general
+    vmImage: MMSUbuntu18.04
+  displayName: Generate Job Matrix
+  steps:
+    - ${{ each config in parameters.MatrixConfigs }}:
+      - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+        - pwsh: |
+            Write-Host "DISPLAY NAME FILTER: $(displayNameFilter)"
+            eng/scripts/job-matrix/Create-JobMatrix.ps1 `
+              -ConfigPath ${{ config.Path }} `
+              -Selection ${{ config.Selection }} `
+              -DisplayNameFilter "$(displayNameFilter)" `
+              -Filters @("container=^$", "SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}")
+          name: generate_vm_job_matrix_${{ config.Name }}
+          displayName: Generate VM Job Matrix
+
+      - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+        - pwsh: |
+            Write-Host "DISPLAY NAME FILTER: $(displayNameFilter)"
+            eng/scripts/job-matrix/Create-JobMatrix.ps1 `
+              -ConfigPath ${{ config.Path }} `
+              -Selection ${{ config.Selection }} `
+              -DisplayNameFilter "$(displayNameFilter)" `
+              -Filters @("container=.*", "SupportedClouds=$^$|{{ parameters.CloudConfig.Cloud }}")
+          name: generate_container_job_matrix_${{ config.Name }}
+          displayName: Generate Container Job Matrix
+
+- ${{ each config in parameters.MatrixConfigs }}:
+  - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+    - template: ./archetype-sdk-tests-jobs.yml
+      parameters:
+        UsePlatformContainer: false
+        Matrix: dependencies.generate_matrix.outputs['generate_vm_job_matrix_${{ config.Name }}.matrix']
+        DependsOn: generate_matrix
+        CloudConfig: ${{ parameters.CloudConfig }}
+        ${{ each param in parameters.AdditionalParameters }}:
+          ${{ param.key }}: ${{ param.value }}
+
+  - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+    - template: ./archetype-sdk-tests-jobs.yml
+      parameters:
+        UsePlatformContainer: true
+        Matrix: dependencies.generate_matrix.outputs['generate_container_job_matrix_${{ config.Name }}.matrix']
+        DependsOn: generate_matrix
+        CloudConfig: ${{ parameters.CloudConfig }}
+        ${{ each param in parameters.AdditionalParameters }}:
+          ${{ param.key }}: ${{ param.value }}

eng/pipelines/templates/jobs/archetype-sdk-tests-host.yml

@@ -35,15 +35,22 @@ parameters:
 - name: Platforms
   type: object
   default: {}
+- name: Matrix
+  type: string
+- name: DependsOn
+  type: string
+  default: ''
 - name: UsePlatformContainer
   type: boolean
   default: false
 
 jobs:
   - job:
+    dependsOn: ${{ parameters.DependsOn }}
+    condition: ne(${{ parameters.Matrix }}, '{}')
     strategy:
       maxParallel: ${{ parameters.MaxParallel }}
-      matrix: ${{ parameters.Platforms }}
+      matrix: $[ ${{ parameters.Matrix }} ]
 
     variables:
       - template: ../variables/globals.yml
@@ -64,6 +71,7 @@ jobs:
       vmImage: $(OSVmImage)
 
     ${{ if eq(parameters.UsePlatformContainer, 'true') }}:
+      # Add a default so the job doesn't fail when the matrix is empty
       container: $[ variables['Container'] ]
 
     steps:

eng/pipelines/templates/jobs/archetype-sdk-tests-jobs.yml

@@ -52,46 +52,16 @@ parameters:
       SubscriptionConfiguration: $(sub-config-gov-test-resources)
     China:
       SubscriptionConfiguration: $(sub-config-cn-test-resources)
-- name: Platforms
+- name: MatrixConfigs
   type: object
   default:
-    Linux:
-      Pool: azsdk-pool-mms-ubuntu-1804-general
-      OSVmImage: MMSUbuntu18.04
-      TestTargetFramework: netcoreapp2.1
-    Windows_NetCoreApp:
-      Pool: azsdk-pool-mms-win-2019-general
-      OSVmImage: MMS2019
-      TestTargetFramework: netcoreapp2.1
-      SupportsRecording: true
-    Windows_NetCoreApp_ProjectReferences:
-      Pool: azsdk-pool-mms-win-2019-general
-      OSVmImage: MMS2019
-      TestTargetFramework: netcoreapp2.1
-      AdditionalTestArguments: "/p:UseProjectReferenceToAzureClients=true"
-    Windows_NetFramework:
-      Pool: azsdk-pool-mms-win-2019-general
-      OSVmImage: MMS2019
-      TestTargetFramework: net461
-    Windows_NetFramework_ProjectReferences:
-      Pool: azsdk-pool-mms-win-2019-general
-      OSVmImage: MMS2019
-      TestTargetFramework: net461
-      AdditionalTestArguments: "/p:UseProjectReferenceToAzureClients=true"
-    MacOS:
-      Pool: Azure Pipelines
-      OSVmImage: "macOS-10.15"
-      TestTargetFramework: netcoreapp2.1
-      SupportedClouds: 'Public'
-    Windows_Net50:
-      Pool: azsdk-pool-mms-win-2019-general
-      OSVmImage: MMS2019
-      TestTargetFramework: net5.0
-    Windows_Net50_ProjectReferences:
-      Pool: azsdk-pool-mms-win-2019-general
-      OSVmImage: MMS2019
-      TestTargetFramework: net5.0
-      AdditionalTestArguments: "/p:UseProjectReferenceToAzureClients=true"
+    - Name: NET_live_test_base
+      Path: eng/pipelines/templates/stages/platform-matrix.json
+      Selection: sparse
+      GenerateVMJobs: true
+- name: AdditionalMatrixConfigs
+  type: object
+  default: []
 - name: PlatformPreSteps
   type: object
   default:
@@ -126,13 +96,8 @@ stages:
       - stage: ${{ cloud.key }}
         dependsOn: []
         jobs:
-        - template: ../jobs/archetype-sdk-tests-host.yml
+        - template: /eng/pipelines/templates/jobs/archetype-sdk-tests-generate.yml
           parameters:
-            # Flag to include the job template with a container field
-            ${{ each platform in parameters.AdditionalPlatforms }}:
-              ${{ if or(not(platform.value.SupportedClouds), contains(platform.value.SupportedClouds, cloud.key)) }}:
-                ${{ if platform.value.Container }}:
-                  UsePlatformContainer: true
             AdditionalParameters:
               PreSteps:
                 - ${{ parameters.PlatformPreSteps }}
@@ -148,14 +113,13 @@ stages:
               SDKType: ${{ parameters.SDKType }}
               ServiceDirectory: ${{ parameters.ServiceDirectory }}
               TestSetupSteps: ${{ parameters.TestSetupSteps }}
-            Platforms:
+            MatrixConfigs:
               # Enumerate platforms and additional platforms based on supported clouds (sparse platform<-->cloud matrix).
-              ${{ each platform in parameters.Platforms }}:
-                ${{ if or(not(platform.value.SupportedClouds), contains(platform.value.SupportedClouds, cloud.key)) }}:
-                  ${{ platform.key }}: ${{ platform.value }}
-              ${{ each platform in parameters.AdditionalPlatforms }}:
-                ${{ if or(not(platform.value.SupportedClouds), contains(platform.value.SupportedClouds, cloud.key)) }}:
-                  ${{ platform.key }}: ${{ platform.value }}
+              - ${{ each config in parameters.MatrixConfigs }}:
+                -  ${{ config }}
+              - ${{ each config in parameters.AdditionalMatrixConfigs }}:
+                -  ${{ config }}
             CloudConfig:
               SubscriptionConfiguration: ${{ cloud.value.SubscriptionConfiguration }}
               Location: ${{ coalesce(parameters.Location, cloud.value.Location) }}
+              Cloud: ${{ cloud.key }}

eng/pipelines/templates/stages/archetype-sdk-tests.yml

@@ -0,0 +1,30 @@
+{
+  "displayNames": {
+    "/p:UseProjectReferenceToAzureClients=true": "UseProjectRef",
+    "true": "Record"
+  },
+  "matrix": {
+    "Agent": {
+      "ubuntu-18.04": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" },
+      "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" },
+      "macOS-10.15": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" }
+    },
+    "TestTargetFramework": [ "netcoreapp2.1", "net5.0" ]
+  },
+  "include": [
+    {
+      "Agent": {
+        "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" }
+      },
+      "TestTargetFramework": [ "net461", "net5.0" ],
+      "AdditionalTestArguments": "/p:UseProjectReferenceToAzureClients=true"
+    },
+    {
+      "Agent": {
+        "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" }
+      },
+      "TestTargetFramework": "netcoreapp2.1",
+      "SupportsRecording": true
+    }
+  ]
+}

eng/pipelines/templates/stages/platform-matrix.json

@@ -10,14 +10,16 @@
 [CmdletBinding()]
 param (
     [Parameter(Mandatory=$True)][string] $ConfigPath,
-    [Parameter(Mandatory=$True)][string] $Selection
+    [Parameter(Mandatory=$True)][string] $Selection,
+    [Parameter(Mandatory=$False)][string] $DisplayNameFilter,
+    [Parameter(Mandatory=$False)][array] $Filters
 )
 
-Import-Module $PSScriptRoot/job-matrix-functions.psm1
+. $PSScriptRoot/job-matrix-functions.ps1
 
 $config = GetMatrixConfigFromJson (Get-Content $ConfigPath)
 
-[array]$matrix = GenerateMatrix $config $Selection
+[array]$matrix = GenerateMatrix $config $Selection $DisplayNameFilter $Filters
 $serialized = SerializePipelineMatrix $matrix
 
 Write-Output $serialized.pretty

eng/scripts/job-matrix/Create-JobMatrix.ps1

@@ -38,20 +38,35 @@ jobs:
       - Name: base_product_matrix
         Path: /eng/pipelines/matrix.json
         Selection: sparse
+		GenerateVMJobs: true
       - Name: sdk_specific_matrix
         Path: /sdk/foobar/matrix.json
         Selection: all
+		GenerateContainerJobs: true
 	steps:
 	  - pwsh:
           ...
 ```
 
 ## Matrix config file syntax
 
+Matrix parameters can either be a list of strings, or a set of grouped strings (represented as a hash). The latter parameter
+type is useful for when 2 or more parameters need to be grouped together, but without generating more than one matrix permutation.
+
 ```
 "matrix": {
   "<parameter1 name>": [ <values...> ],
-  "<parameter2 name>": [ <values...> ]
+  "<parameter2 name>": [ <values...> ],
+  "<parameter set>": {
+	"<parameter set 1 name>": {
+		"<parameter set 1 value 1": "value",
+		"<parameter set 1 value 2": "<value>",
+	},
+	"<parameter set 2 name>": {
+		"<parameter set 2 value 1": "value",
+		"<parameter set 2 value 2": "<value>",
+	}
+  }
 }
 "include": [ <matrix>, <matrix>, ... ],
 "exclude": [ <matrix>, <matrix>, ... ],
@@ -202,19 +217,30 @@ The top level keys are used as job names, meaning they get displayed in the azur
 
 The logic for generating display names works like this:
 
-1. Sort the matrix by: parameter length, parameter name, and sort the parameter values arrays.
-2. In sorted order, join parameter values by "_"
+- Join parameter values by "_"
     a. If the parameter value exists as a key in `displayNames` in the matrix config, replace it with that value.
-    b. For each name value, strip all `.-_` characters from the string.
-
-The matrix is then sorted by display name, before being sent to azure pipelines. The underlying matrix may have a different
-sorted order than the display name output. The sorting needs to be separate so that sparse matrix generation can be deterministic.
+    b. For each name value, strip all non-alphanumeric characters (excluding "_").
+	c. If the name is greater than 100 characters, truncate it.
 
 #### Filters
 
-To be implemented. A basic example of a filter is "all" vs. "sparse" generation, but eventually will be a more programmable
-way of processing excludes, such as an expression that only includes entries with a container image specified. The intent
-is that these filters can be entered at runtime, as opposed to statically in yaml.
+Filters can be passed to the matrix as an array of strings, each matching the format of <key>=<regex>. When a matrix entry
+does not contain the specified key, it will default to a value of empty string for regex parsing. This can be used to specify
+filters for keys that don't exist or keys that optionally exist and match a regex, as seen in the below example.
+
+Display name filters can also be passed as a single regex string that runs against the [generated display name](#displaynames) of the matrix job.
+The intent of display name filters is to be defined primarily as a top level variable at template queue time in the azure pipelines UI.
+
+For example, the below command will filter for matrix entries with "windows" in the job display name, no matrix variable
+named "ExcludedKey", a framework variable containing either "461" or "5.0", and an optional key "SupportedClouds" that, if exists, must contain "Public":
+
+```
+./Create-JobMatrix.ps1 `
+  -ConfigPath samples/matrix.json `
+  -Selection all `
+  -DisplayNameFilter ".*windows.*" `
+  -Filters @("ExcludedKey=^$", "framework=(461|5\.0)", "SupportedClouds=^$|.*Public.*")
+```
 
 #### Under the hood