Add storage test pipeline with static config for virtualized resource (Azure#29916)

Author: benbp

eng/pipelines/templates/jobs/live.tests.yml

@@ -32,6 +32,9 @@ parameters:
 - name: TestSetupSteps
   type: stepList
   default: []
+- name: DeployTestResources
+  type: boolean
+  default: true
 - name: CloudConfig
   type: object
   default: {}
@@ -99,13 +102,14 @@ jobs:
           SubscriptionConfiguration: ${{ parameters.CloudConfig.SubscriptionConfiguration }}
           SubscriptionConfigurations: ${{ parameters.CloudConfig.SubscriptionConfigurations }}
 
-      - template: /eng/common/TestResources/deploy-test-resources.yml
-        parameters:
-          ${{ if or(parameters.Location, parameters.CloudConfig.Location) }}:
-            Location: ${{ coalesce(parameters.Location, parameters.CloudConfig.Location) }}
-          ServiceDirectory: '${{ parameters.ServiceDirectory }}'
-          SubscriptionConfiguration: $(SubscriptionConfiguration)
-          ArmTemplateParameters: $(ArmTemplateParameters)
+      - ${{ if parameters.DeployTestResources }}:
+        - template: /eng/common/TestResources/deploy-test-resources.yml
+          parameters:
+            ${{ if or(parameters.Location, parameters.CloudConfig.Location) }}:
+              Location: ${{ coalesce(parameters.Location, parameters.CloudConfig.Location) }}
+            ServiceDirectory: '${{ parameters.ServiceDirectory }}'
+            SubscriptionConfiguration: $(SubscriptionConfiguration)
+            ArmTemplateParameters: $(ArmTemplateParameters)
 
       - pwsh: |
           if ($env:SupportsRecording -and $env:Record) {
@@ -138,10 +142,11 @@ jobs:
           ${{ each var in parameters.EnvVars }}:
             ${{ var.key }}: ${{ var.value }}
 
-      - template: /eng/common/TestResources/remove-test-resources.yml
-        parameters:
-          ServiceDirectory: '${{ parameters.ServiceDirectory }}'
-          SubscriptionConfiguration: $(SubscriptionConfiguration)
+      - ${{ if parameters.DeployTestResources }}:
+        - template: /eng/common/TestResources/remove-test-resources.yml
+          parameters:
+            ServiceDirectory: '${{ parameters.ServiceDirectory }}'
+            SubscriptionConfiguration: $(SubscriptionConfiguration)
 
       - task: PublishTestResults@2
         condition: always()

eng/pipelines/templates/stages/archetype-sdk-tests.yml

@@ -44,6 +44,9 @@ parameters:
 - name: TestSetupSteps
   type: stepList
   default: []
+- name: DeployTestResources
+  type: boolean
+  default: true
 - name: CloudConfig
   type: object
   default:
@@ -128,6 +131,7 @@ stages:
               ServiceDirectory: ${{ parameters.ServiceDirectory }}
               Project: ${{ parameters.Project }}
               TestSetupSteps: ${{ parameters.TestSetupSteps }}
+              DeployTestResources: ${{ parameters.DeployTestResources }}
             MatrixConfigs:
               # Enumerate platforms and additional platforms based on supported clouds (sparse platform<-->cloud matrix).
               - ${{ each config in parameters.MatrixConfigs }}:

sdk/storage/platform-matrix-virtualized.json

@@ -0,0 +1,20 @@
+{
+  "displayNames": {
+    "/p:UseProjectReferenceToAzureClients=false": "PackageRef"
+  },
+  "matrix": {
+    "Agent": {
+      "Ubuntu-20.04_NET6.0": {
+        "OSVmImage": "MMSUbuntu20.04",
+        "Pool": "azsdk-pool-mms-ubuntu-2004-general",
+        "TestTargetFramework": "net6.0"
+      }
+    },
+    "AdditionalTestArguments": [
+        "/p:UseProjectReferenceToAzureClients=false"
+    ],
+    "BuildConfiguration": [
+        "Debug"
+    ]
+  }
+}

sdk/storage/set-test-configuration.ps1

@@ -0,0 +1,35 @@
+#!/usr/bin/env pwsh
+
+# This script enables bypassing ARM template deployment in live tests in favor of a statically configured
+# test configuration xml file stored in keyvault. The test configuration file is normally constructed
+# dynamically from ARM template outputs via test-resources-post.ps1
+[CmdletBinding()]
+param (
+    [ValidateNotNullOrEmpty()]
+    [string] $TestConfigurationXmlContent
+)
+
+function getTestConfigurationPath() {
+    $storageTestConfigurationTemplateName = 'TestConfigurationsTemplate.xml'
+    $storageTestConfigurationName = 'TestConfigurations.xml'
+
+    if(-not (Test-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY -ErrorAction Ignore) ) {
+      Write-Verbose "Checking for '$storageTestConfigurationTemplateName' files under '$PSScriptRoot'"
+
+      $foundFile = Get-ChildItem -Path $PSScriptRoot -Filter $storageTestConfigurationTemplateName -Recurse | Select-Object -First 1
+      $storageTemplateDirName = $foundFile.Directory.FullName
+      Write-Verbose "Found template dir '$storageTemplateDirName'"
+
+    } else {
+      $storageTemplateDirName = $env:BUILD_ARTIFACTSTAGINGDIRECTORY
+      Write-Verbose "Found environment variable BUILD_ARTIFACTSTAGINGDIRECTORY '$storageTemplateDirName'"
+    }
+
+    # Construct the test configuration path to use based on the devops build variable for artifact staging directory
+    $TestConfigurationPath = Join-Path -Path $storageTemplateDirName -ChildPath $storageTestConfigurationName
+    return $TestConfigurationPath
+}
+
+$outfile = getTestConfigurationPath
+Write-Host "Writing test configuration xml to $outfile"
+$TestConfigurationXmlContent | Out-File $outfile

sdk/storage/tests.virtualized.yml

@@ -0,0 +1,25 @@
+trigger: none
+
+extends:
+  template: /eng/pipelines/templates/stages/archetype-sdk-tests.yml
+  parameters:
+    ServiceDirectory: storage
+    MatrixReplace:
+      # Use dedicated storage pool in canadacentral with higher memory capacity
+      - Pool=(.*)-general/$1-storage
+    MatrixConfigs:
+      - Name: Storage_virtualized
+        Path: sdk/storage/platform-matrix-virtualized.json
+        Selection: sparse
+        GenerateVMJobs: true
+    EnvVars:
+      AZURE_ONLY_TEST_LATEST_SERVICE_VERSION: true
+    DeployTestResources: false
+    TestSetupSteps:
+      - template: /sdk/storage/tests-install-azurite.yml
+      - pwsh: |
+          $config = @'
+            $(live-test-config-net-virtualized-xml)
+          '@
+          sdk/storage/set-test-configuration.ps1 -TestConfigurationXmlContent $config
+        displayName: Set Static Test Configuration