Add import and non-sparse parameter functionality to job matrix generation (Azure#18660)

Author: benbp

eng/scripts/job-matrix/Create-JobMatrix.ps1

@@ -12,14 +12,23 @@ param (
     [Parameter(Mandatory=$True)][string] $ConfigPath,
     [Parameter(Mandatory=$True)][string] $Selection,
     [Parameter(Mandatory=$False)][string] $DisplayNameFilter,
-    [Parameter(Mandatory=$False)][array] $Filters
+    [Parameter(Mandatory=$False)][array] $Filters,
+    [Parameter(Mandatory=$False)][array] $NonSparseParameters
 )
 
 . $PSScriptRoot/job-matrix-functions.ps1
 
 $config = GetMatrixConfigFromJson (Get-Content $ConfigPath)
+# Strip empty string filters in order to be able to use azure pipelines yaml join()
+$Filters = $Filters | Where-Object { $_ }
+
+[array]$matrix = GenerateMatrix `
+    -config $config `
+    -selectFromMatrixType $Selection `
+    -displayNameFilter $DisplayNameFilter `
+    -filters $Filters `
+    -nonSparseParameters $NonSparseParameters
 
-[array]$matrix = GenerateMatrix $config $Selection $DisplayNameFilter $Filters
 $serialized = SerializePipelineMatrix $matrix
 
 Write-Output $serialized.pretty

eng/scripts/job-matrix/README.md

@@ -2,18 +2,20 @@
 
 * [Usage in a pipeline](#usage-in-a-pipeline)
 * [Matrix config file syntax](#matrix-config-file-syntax)
-   * [Fields](#fields)
-	  * [matrix](#matrix)
-	  * [include](#include)
-	  * [exclude](#exclude)
-	  * [displayNames](#displaynames)
+ * [Fields](#fields)
+    * [matrix](#matrix)
+    * [include](#include)
+    * [exclude](#exclude)
+    * [displayNames](#displaynames)
+    * [$IMPORT](#import)
 * [Matrix Generation behavior](#matrix-generation-behavior)
-  * [all](#all)
-  * [sparse](#sparse)
-  * [include/exclude](#includeexclude)
-  * [displayNames](#displaynames-1)
-  * [Filters](#filters)
-  * [Under the hood](#under-the-hood)
+    * [all](#all)
+    * [sparse](#sparse)
+    * [include/exclude](#includeexclude)
+    * [displayNames](#displaynames-1)
+    * [Filters](#filters)
+    * [NonSparseParameters](#nonsparseparameters)
+    * [Under the hood](#under-the-hood)
 * [Testing](#testing)
 
 
@@ -38,13 +40,14 @@ jobs:
       - Name: base_product_matrix
         Path: /eng/pipelines/matrix.json
         Selection: sparse
-		GenerateVMJobs: true
+        NonSparseParameters: <csv of parameter names for which all combinations should be included>
+        GenerateVMJobs: true
       - Name: sdk_specific_matrix
         Path: /sdk/foobar/matrix.json
         Selection: all
-		GenerateContainerJobs: true
-	steps:
-	  - pwsh:
+        GenerateContainerJobs: true
+    steps:
+      - pwsh:
           ...
 ```
 
@@ -58,14 +61,14 @@ type is useful for when 2 or more parameters need to be grouped together, but wi
   "<parameter1 name>": [ <values...> ],
   "<parameter2 name>": [ <values...> ],
   "<parameter set>": {
-	"<parameter set 1 name>": {
-		"<parameter set 1 value 1": "value",
-		"<parameter set 1 value 2": "<value>",
-	},
-	"<parameter set 2 name>": {
-		"<parameter set 2 value 1": "value",
-		"<parameter set 2 value 2": "<value>",
-	}
+    "<parameter set 1 name>": {
+        "<parameter set 1 value 1": "value",
+        "<parameter set 1 value 2": "<value>",
+    },
+    "<parameter set 2 name>": {
+        "<parameter set 2 value 1": "value",
+        "<parameter set 2 value 2": "<value>",
+    }
   }
 }
 "include": [ <matrix>, <matrix>, ... ],
@@ -144,6 +147,150 @@ readable value here. For example:
 }
 ```
 
+#### $IMPORT
+
+Matrix configs can also import another matrix config. The effect of this is the imported matrix will be generated,
+and then the importing config will be combined with that matrix (as if each entry of the imported matrix was a parameter).
+To import a matrix, add a parameter with the key `$IMPORT`:
+
+```
+"matrix": {
+  "$IMPORT": "path/to/matrix.json",
+  "JavaVersion": [ "1.8", "1.11" ]
+}
+```
+
+Importing can be useful, for example, in cases where there is a shared base matrix, but there is a need to run it
+once for each instance of a language version.
+
+The processing order is as follows:
+
+Given a matrix and import matrix like below:
+```
+{
+    "matrix": {
+        "$IMPORT": "example-matrix.json",
+        "endpointType": [ "storage", "cosmos" ],
+        "JavaVersion": [ "1.8", "1.11" ]
+    },
+    "include": [
+        {
+            "operatingSystem": "windows",
+            "mode": "TestFromSource",
+            "JavaVersion": "1.8"
+        }
+    ]
+}
+
+### example-matrix.json to import
+{
+    "matrix": {
+      "operatingSystem": [ "windows", "linux" ],
+      "client": [ "netty", "okhttp" ]
+    },
+    "include": [
+        {
+          "operatingSystem": "mac",
+          "client": "netty"
+        }
+    ]
+}
+```
+
+1. The base matrix is generated (sparse in this example):
+    ```
+    {
+      "storage_18": {
+        "endpointType": "storage",
+        "JavaVersion": "1.8"
+      },
+      "cosmos_111": {
+        "endpointType": "cosmos",
+        "JavaVersion": "1.11"
+      }
+    }
+    ```
+1. The imported base matrix is generated (sparse in this example):
+    ```
+    {
+      "windows_netty": {
+        "operatingSystem": "windows",
+        "client": "netty"
+      },
+      "linux_okhttp": {
+        "operatingSystem": "linux",
+        "client": "okhttp"
+      }
+    }
+    ```
+1. Includes/excludes from the imported matrix get applied to the imported matrix
+    ```
+    {
+      "windows_netty": {
+        "operatingSystem": "windows",
+        "client": "netty"
+      },
+      "linux_okhttp": {
+        "operatingSystem": "linux",
+        "client": "okhttp"
+      },
+      "mac_netty": {
+        "operatingSystem": "mac",
+        "client": "netty"
+      }
+    }
+    ```
+1. The base matrix is multipled by the imported matrix (in this case, the base matrix has 2 elements, and the imported
+   matrix has 3 elements, so the product is a matrix with 6 elements:
+    ```
+      "storage_18_windows_netty": {
+        "endpointType": "storage",
+        "JavaVersion": "1.8",
+        "operatingSystem": "windows",
+        "client": "netty"
+      },
+      "storage_18_linux_okhttp": {
+        "endpointType": "storage",
+        "JavaVersion": "1.8",
+        "operatingSystem": "linux",
+        "client": "okhttp"
+      },
+      "storage_18_mac_netty": {
+        "endpointType": "storage",
+        "JavaVersion": "1.8",
+        "operatingSystem": "mac",
+        "client": "netty"
+      },
+      "cosmos_111_windows_netty": {
+        "endpointType": "cosmos",
+        "JavaVersion": "1.11",
+        "operatingSystem": "windows",
+        "client": "netty"
+      },
+      "cosmos_111_linux_okhttp": {
+        "endpointType": "cosmos",
+        "JavaVersion": "1.11",
+        "operatingSystem": "linux",
+        "client": "okhttp"
+      },
+      "cosmos_111_mac_netty": {
+        "endpointType": "cosmos",
+        "JavaVersion": "1.11",
+        "operatingSystem": "mac",
+        "client": "netty"
+      }
+    }
+    ```
+1. Includes/excludes from the top-level matrix get applied to the multiplied matrix, so the below element will be added
+   to the above matrix, for an output matrix with 7 elements:
+    ```
+    "windows_TestFromSource_18": {
+      "operatingSystem": "windows",
+      "mode": "TestFromSource",
+      "JavaVersion": "1.8"
+    }
+    ```
+
 ## Matrix Generation behavior
 
 #### all
@@ -220,7 +367,7 @@ The logic for generating display names works like this:
 - Join parameter values by "_"
     a. If the parameter value exists as a key in `displayNames` in the matrix config, replace it with that value.
     b. For each name value, strip all non-alphanumeric characters (excluding "_").
-	c. If the name is greater than 100 characters, truncate it.
+    c. If the name is greater than 100 characters, truncate it.
 
 #### Filters
 
@@ -242,6 +389,38 @@ named "ExcludedKey", a framework variable containing either "461" or "5.0", and
   -Filters @("ExcludedKey=^$", "framework=(461|5\.0)", "SupportedClouds=^$|.*Public.*")
 ```
 
+#### NonSparseParameters
+
+Sometimes it may be necessary to generate a sparse matrix, but keep the full combination of a few parameters. The
+NonSparseParameters argument allows for more fine-grained control of matrix generation. For example:
+
+```
+./Create-JobMatrix.ps1 `
+  -ConfigPath /path/to/matrix.json `
+  -Selection sparse `
+  -NonSparseParameters @("JavaTestVersion")
+```
+
+Given a matrix like below with `JavaTestVersion` marked as a non-sparse parameter:
+
+```
+{
+  "matrix": {
+    "Agent": {
+      "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" },
+      "ubuntu-1804": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" },
+      "macOS-10.15": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" }
+    },
+    "JavaTestVersion": [ "1.8", "1.11" ],
+    "AZURE_TEST_HTTP_CLIENTS": "netty",
+    "ArmTemplateParameters": [ "@{endpointType='storage'}", "@{endpointType='cosmos'}" ]
+  }
+}
+```
+
+A matrix with 6 entries will be generated: A sparse matrix of Agent, AZURE_TEST_HTTP_CLIENTS and ArmTemplateParameters
+(3 total entries) will be multipled by the two `JavaTestVersion` parameters `1.8` and `1.11`.
+
 #### Under the hood
 
 The script generates an N-dimensional matrix with dimensions equal to the parameter array lengths. For example,