Sync eng/common directory with azure-sdk-tools for PR 1163 (Azure#12346)

* Update subscription configuration schema to include new parameters

* Support platform specific arm template parameters and legacy hashtable format

* Update arm template parameter comment to include top level key

* Restore AdditionalParameters. Merge ArmTemplateParameters from stringified hash literal

* Handle duplicate keys more explicitly for arm and env vars

* Regenerate New-TestResources.ps1 markdown

* revert variable name to environmentVariables to fix post-scripts

* Handle empty arm template parameters better

* Remove arm template parameter merge logic from deploy template

* Add merge hashes function to New-TestResources.ps1

* Add merge hashes function to New-TestResources.ps1

* Add env variable overwrite warning. Use ContainsKey checks

* Temporarily manually fix invalid generated markdown links

Co-authored-by: Ben Broderick Phillips <bebroder@microsoft.com>

Author: azure-sdk

eng/common/TestResources/New-TestResources.ps1

@@ -10,7 +10,7 @@
 
 [CmdletBinding(DefaultParameterSetName = 'Default', SupportsShouldProcess = $true, ConfirmImpact = 'Medium')]
 param (
-    # Limit $BaseName to enough characters to be under limit plus prefixes, and https://docs.microsoft.com/azure/architecture/best-practices/resource-naming.
+    # Limit $BaseName to enough characters to be under limit plus prefixes, and https://docs.microsoft.com/azure/architecture/best-practices/resource-naming
     [Parameter()]
     [ValidatePattern('^[-a-zA-Z0-9\.\(\)_]{0,80}(?<=[a-zA-Z0-9\(\)])$')]
     [string] $BaseName,
@@ -58,9 +58,16 @@ param (
     [ValidateSet('AzureCloud', 'AzureUSGovernment', 'AzureChinaCloud', 'Dogfood')]
     [string] $Environment = 'AzureCloud',
 
+    [Parameter()]
+    [hashtable] $ArmTemplateParameters,
+
     [Parameter()]
     [hashtable] $AdditionalParameters,
 
+    [Parameter()]
+    [ValidateNotNull()]
+    [hashtable] $EnvironmentVariables = @{},
+
     [Parameter()]
     [switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID),
 
@@ -101,6 +108,16 @@ function Retry([scriptblock] $Action, [int] $Attempts = 5) {
     }
 }
 
+function MergeHashes([hashtable] $source, [psvariable] $dest) {
+    foreach ($key in $source.Keys) {
+        if ($dest.Value.ContainsKey($key) -and $dest.Value[$key] -ne $source[$key]) {
+            Write-Warning ("Overwriting '$($dest.Name).$($key)' with value '$($dest.Value[$key])' " +
+                          "to new value '$($source[$key])'")
+        }
+        $dest.Value[$key] = $source[$key]
+    }
+}
+
 # Support actions to invoke on exit.
 $exitActions = @({
     if ($exitActions.Count -gt 1) {
@@ -118,7 +135,6 @@ $repositoryRoot = "$PSScriptRoot/../../.." | Resolve-Path
 $root = [System.IO.Path]::Combine($repositoryRoot, "sdk", $ServiceDirectory) | Resolve-Path
 $templateFileName = 'test-resources.json'
 $templateFiles = @()
-$environmentVariables = @{}
 # Azure SDK Developer Playground
 $defaultSubscription = "faa080af-c1d8-40ad-9cce-e1a450ca5b57"
 
@@ -252,7 +268,7 @@ $serviceName = if (Split-Path -IsAbsolute  $ServiceDirectory) {
     $ServiceDirectory
 }
 
-if ($CI) { 
+if ($CI) {
   $BaseName = 't' + (New-Guid).ToString('n').Substring(0, 16)
   Write-Verbose "Generated base name '$BaseName' for CI build"
 }
@@ -289,7 +305,13 @@ if ($CI) {
     # Set the resource group name variable.
     Write-Host "Setting variable 'AZURE_RESOURCEGROUP_NAME': $ResourceGroupName"
     Write-Host "##vso[task.setvariable variable=AZURE_RESOURCEGROUP_NAME;]$ResourceGroupName"
-    $environmentVariables['AZURE_RESOURCEGROUP_NAME'] = $ResourceGroupName
+    if ($EnvironmentVariables.ContainsKey('AZURE_RESOURCEGROUP_NAME') -and `
+        $EnvironmentVariables['AZURE_RESOURCEGROUP_NAME'] -ne $ResourceGroupName)
+    {
+        Write-Warning ("Overwriting 'EnvironmentVariables.AZURE_RESOURCEGROUP_NAME' with value " + 
+            "'$($EnvironmentVariables['AZURE_RESOURCEGROUP_NAME'])' " + "to new value '$($ResourceGroupName)'")
+    }
+    $EnvironmentVariables['AZURE_RESOURCEGROUP_NAME'] = $ResourceGroupName
 }
 
 Log "Creating resource group '$ResourceGroupName' in location '$Location'"
@@ -322,11 +344,11 @@ if ($TenantId) {
 if ($TestApplicationSecret) {
     $templateParameters.Add('testApplicationSecret', $TestApplicationSecret)
 }
-if ($AdditionalParameters) {
-    $templateParameters += $AdditionalParameters
-}
 
-# Include environment-specific parameters only if not already provided as part of the "AdditionalParameters"
+MergeHashes $ArmTemplateParameters $(Get-Variable templateParameters)
+MergeHashes $AdditionalParameters $(Get-Variable templateParameters)
+
+# Include environment-specific parameters only if not already provided as part of the "ArmTemplateParameters"
 if (($context.Environment.StorageEndpointSuffix) -and (-not ($templateParameters.ContainsKey('storageEndpointSuffix')))) {
     $templateParameters.Add('storageEndpointSuffix', $context.Environment.StorageEndpointSuffix)
 }
@@ -388,6 +410,8 @@ foreach ($templateFile in $templateFiles) {
         "$($serviceDirectoryPrefix)STORAGE_ENDPOINT_SUFFIX" = $context.Environment.StorageEndpointSuffix;
     }
 
+    MergeHashes $EnvironmentVariables $(Get-Variable deploymentOutputs)
+
     foreach ($key in $deployment.Outputs.Keys) {
         $variable = $deployment.Outputs[$key]
 
@@ -422,7 +446,7 @@ foreach ($templateFile in $templateFiles) {
 
         foreach ($key in $deploymentOutputs.Keys) {
             $value = $deploymentOutputs[$key]
-            $environmentVariables[$key] = $value
+            $EnvironmentVariables[$key] = $value
 
             if ($CI) {
                 # Treat all ARM template output variables as secrets since "SecureString" variables do not set values.
@@ -453,7 +477,7 @@ $exitActions.Invoke()
 
 # Suppress output locally
 if ($CI) {
-    return $environmentVariables
+    return $EnvironmentVariables
 }
 
 <#
@@ -571,8 +595,14 @@ Name of the cloud environment. The default is the Azure Public Cloud
 ('AzureCloud')
 
 .PARAMETER AdditionalParameters
+Optional key-value pairs of parameters to pass to the ARM template(s) and pre-post scripts.
+
+.PARAMETER ArmTemplateParameters
 Optional key-value pairs of parameters to pass to the ARM template(s).
 
+.PARAMETER EnvironmentVariables
+Optional key-value pairs of parameters to set as environment variables to the shell.
+
 .PARAMETER CI
 Indicates the script is run as part of a Continuous Integration / Continuous
 Deployment (CI/CD) build (only Azure Pipelines is currently supported).
@@ -617,6 +647,4 @@ Run this in an Azure DevOps CI (with approrpiate variables configured) before
 executing live tests. The script will output variables as secrets (to enable
 log redaction).
 
-.LINK
-Remove-TestResources.ps1
 #>

eng/common/TestResources/New-TestResources.ps1.md

@@ -16,8 +16,9 @@ Deploys live test resources defined for a service directory to Azure.
 ```
 New-TestResources.ps1 [-BaseName <String>] [-ResourceGroupName <String>] [-ServiceDirectory] <String>
  [-TestApplicationId <String>] [-TestApplicationSecret <String>] [-TestApplicationOid <String>]
- [-DeleteAfterHours <Int32>] [-Location <String>] [-Environment <String>] [-AdditionalParameters <Hashtable>]
- [-CI] [-Force] [-OutFile] [-WhatIf] [-Confirm] [<CommonParameters>]
+ [-DeleteAfterHours <Int32>] [-Location <String>] [-Environment <String>] [-ArmTemplateParameters <Hashtable>]
+ [-AdditionalParameters <Hashtable>] [-EnvironmentVariables <Hashtable>] [-CI] [-Force] [-OutFile] [-WhatIf]
+ [-Confirm] [<CommonParameters>]
 ```
 
 ### Provisioner
@@ -26,8 +27,8 @@ New-TestResources.ps1 [-BaseName <String>] [-ResourceGroupName <String>] [-Servi
  [-TestApplicationId <String>] [-TestApplicationSecret <String>] [-TestApplicationOid <String>]
  -TenantId <String> [-SubscriptionId <String>] -ProvisionerApplicationId <String>
  -ProvisionerApplicationSecret <String> [-DeleteAfterHours <Int32>] [-Location <String>]
- [-Environment <String>] [-AdditionalParameters <Hashtable>] [-CI] [-Force] [-OutFile] [-WhatIf] [-Confirm]
- [<CommonParameters>]
+ [-Environment <String>] [-ArmTemplateParameters <Hashtable>] [-AdditionalParameters <Hashtable>]
+ [-EnvironmentVariables <Hashtable>] [-CI] [-Force] [-OutFile] [-WhatIf] [-Confirm] [<CommonParameters>]
 ```
 
 ## DESCRIPTION
@@ -339,7 +340,7 @@ Accept wildcard characters: False
 ### -Environment
 Name of the cloud environment.
 The default is the Azure Public Cloud
-('PublicCloud')
+('AzureCloud')
 
 ```yaml
 Type: String
@@ -353,7 +354,7 @@ Accept pipeline input: False
 Accept wildcard characters: False
 ```
 
-### -AdditionalParameters
+### -ArmTemplateParameters
 Optional key-value pairs of parameters to pass to the ARM template(s).
 
 ```yaml
@@ -368,6 +369,36 @@ Accept pipeline input: False
 Accept wildcard characters: False
 ```
 
+### -AdditionalParameters
+Optional key-value pairs of parameters to pass to the ARM template(s) and pre-post scripts.
+
+```yaml
+Type: Hashtable
+Parameter Sets: (All)
+Aliases:
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
+### -EnvironmentVariables
+Optional key-value pairs of parameters to set as environment variables to the shell.
+
+```yaml
+Type: Hashtable
+Parameter Sets: (All)
+Aliases:
+
+Required: False
+Position: Named
+Default value: @{}
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
 ### -CI
 Indicates the script is run as part of a Continuous Integration / Continuous
 Deployment (CI/CD) build (only Azure Pipelines is currently supported).

eng/common/TestResources/Remove-TestResources.ps1

@@ -214,6 +214,4 @@ Remove-TestResources.ps1 `
 When run in the context of an Azure DevOps pipeline, this script removes the
 resource group whose name is stored in the environment variable
 AZURE_RESOURCEGROUP_NAME.
-.LINK
-New-TestResources.ps1
 #>

eng/common/TestResources/deploy-test-resources.yml

@@ -5,7 +5,7 @@ parameters:
   Location: ''
   SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
 
-# SubscriptionConfiguration will be splat into the parameters of the test
+# SubscriptionConfiguration will be splatted into the parameters of the test
 # resources script. It should be JSON in the form:
 # {
 #   "SubscriptionId": "<subscription id>",
@@ -15,25 +15,43 @@ parameters:
 #   "ProvisionerApplicationId": "<provisioner app id>",
 #   "ProvisionerApplicationSecret": "<provisioner app secret>",
 #   "Environment": "AzureCloud | AzureGov | AzureChina | <other environment>"
+#   "EnvironmentVariables": {
+#       "SERVICE_MANAGEMENT_URL": "<service management url>",
+#       "STORAGE_ENDPOINT_SUFFIX": "<storage endpoint suffix>",
+#       "RESOURCE_MANAGER_URL": "<resource manager url>",
+#       "SEARCH_ENDPOINT_SUFFIX": "<search endpoint suffix>",
+#       "COSMOS_TABLES_ENDPOINT_SUFFIX": "<cosmos tables endpoint suffix>"
+#   },
+#   "ArmTemplateParameters": {
+#       "keyVaultDomainSuffix": "<keyVaultDomainSuffix>",
+#       "storageEndpointSuffix": "<storageEndpointSuffix>",
+#       "endpointSuffix": "<endpointSuffix>",
+#       "azureAuthorityHost": "<azureAuthorityHost>",
+#       "keyVaultEndpointSuffix": "<keyVaultEndpointSuffix>"
+#   }
 # }
 
+
 steps:
   - template: /eng/common/TestResources/setup-az-modules.yml
 
   - pwsh: |
       eng/common/TestResources/Import-AzModules.ps1
 
-      $subscriptionConfiguration = @"
+      $subscriptionConfiguration = @'
         ${{ parameters.SubscriptionConfiguration }}
-      "@ | ConvertFrom-Json -AsHashtable;
+      '@ | ConvertFrom-Json -AsHashtable;
 
+      # The subscriptionConfiguration may have ArmTemplateParameters defined, so
+      # pass those in via the ArmTemplateParameters flag, and handle any
+      # additional parameters from the pipelines via AdditionalParameters
       eng/common/TestResources/New-TestResources.ps1 `
         -BaseName 'Generated' `
-        -ServiceDirectory ${{ parameters.ServiceDirectory }} `
+        -ServiceDirectory '${{ parameters.ServiceDirectory }}' `
         -Location '${{ parameters.Location }}' `
-        -DeleteAfterHours ${{ parameters.DeleteAfterHours }} `
-        -AdditionalParameters ${{ parameters.ArmTemplateParameters }} `
+        -DeleteAfterHours '${{ parameters.DeleteAfterHours }}' `
         @subscriptionConfiguration `
+        -AdditionalParameters ${{ parameters.ArmTemplateParameters }} `
         -CI `
         -Force `
         -Verbose | Out-Null