Update live test yaml to use cloud and matrix configuration (Azure#13886)

* Update live test templates to use matrix generation

* Update sdk live tests to use matrix generation, cloud config stages

* Fix live test matrix filter parity errors

* Remove matrix filters. Opt-in most tests to samples and min/max testing

* Fix post step template parameter in monitor live tests

* Filter dependency version for live tests that don't support it

* Only publish test results for browser and node tests

Author: benbp

eng/pipelines/templates/jobs/archetype-sdk-integration.yml

@@ -2,13 +2,7 @@ parameters:
 - name: PackageName
   type: string
   default: ""
-- name: PreIntegrationSteps
-  type: string
-  default: ""
-- name: PostIntegrationSteps
-  type: string
-  default: ""
-- name: ResourceServiceDirectory
+- name: ServiceDirectory
   type: string
   default: ""
 - name: EnvVars
@@ -23,161 +17,64 @@ parameters:
 - name: PublishCodeCoverage
   type: boolean
   default: false
-- name: SubscriptionConfiguration
-  type: string
-  default: $(sub-config-azure-cloud-test-resources)
-- name: ResourceGroupLocation
+- name: PreSteps
+  type: object
+  default: []
+- name: PostSteps
+  type: object
+  default: []
+- name: Location
   type: string
-  default: ""
-- name: ResourceGroupLocationCanary
+  default: ''
+- name: CloudConfig
+  type: object
+  default: {}
+- name: Matrix
   type: string
-  default: "centraluseuap"
-- name: ArmTemplateParameters
+- name: DependsOn
   type: string
-  default: "@{}"
-- name: TestBrowser
-  type: boolean
-  default: true
-- name: TestSamples
-  type: boolean
-  default: true
-- name: TestMinMax
-  type: boolean
-  default: false
-- name: TestSame
+  default: ''
+- name: UsePlatformContainer
   type: boolean
   default: false
-- name: TestCanary
-  type: boolean
-  default: false
-- name: Matrix
-  type: object
-  default:
-    Windows Node 8:
-      Pool: azsdk-pool-mms-win-2019-general
-      OSVmImage: MMS2019
-      TestType: "node"
-      NodeTestVersion: "8.x"
-      PublishCodeCoverage: true
-    Linux Node 10:
-      Pool: azsdk-pool-mms-ubuntu-1804-general
-      OSVmImage: MMSUbuntu18.04
-      TestType: "node"
-      NodeTestVersion: "10.x"
-    Linux Node 12:
-      Pool: azsdk-pool-mms-ubuntu-1804-general
-      OSVmImage: MMSUbuntu18.04
-      TestType: "node"
-      NodeTestVersion: "12.x"
-    MacOS Node 14:
-      Pool: Azure Pipelines
-      OSVmImage: "macOS-10.15"
-      TestType: "node"
-      NodeTestVersion: "14.x"
+
 
 jobs:
-  - job: "IntegrationTest"
+  - job:
+    dependsOn: ${{ parameters.DependsOn }}
+    condition: ne(${{ parameters.Matrix }}, '{}')
+    strategy:
+      maxParallel: ${{ parameters.MaxParallel }}
+      matrix: $[ ${{ parameters.Matrix }} ]
 
     variables:
       - template: ../variables/globals.yml
-      - name: SubscriptionConfiguration
-        value: ${{ parameters.SubscriptionConfiguration }}
-      - name: ResourceGroupLocation
-        value: ${{ parameters.ResourceGroupLocation }}
       - name: ArmTemplateParameters
-        value: ${{ parameters.ArmTemplateParameters }}
-
-
-    strategy:
-      maxParallel: ${{ parameters.MaxParallel }}
-      matrix:
-        ${{ each matrixEntry in parameters.Matrix }}:
-          ${{ matrixEntry.key }}:         
-            # TestResultsFiles
-            ${{ if eq(matrixEntry.value.TestType, 'browser' )}}:
-              TestResultsFiles: "**/test-results.browser.xml"
-            ${{ if eq(matrixEntry.value.TestType, 'node') }}:
-              TestResultsFiles: "**/test-results.xml"
-            # Remaining entries from the matrix object
-            ${{ insert }}: ${{ matrixEntry.value }}
-            # Include these entries in all test matrices. Note that TestResultsFile
-            # is specified because the template logic above is not running in this
-            # context.
-
-        # Add matrix entry for browser testing
-        ${{ if eq(parameters.TestBrowser, 'true') }}:
-          Windows Browser:
-            Pool: azsdk-pool-mms-win-2019-general
-            OSVmImage: MMS2019
-            TestType: "browser"
-            NodeTestVersion: "$(NodeVersion)"
-            TestResultsFiles: "**/test-results.browser.xml"
-
-        # Add matrix entry for sample testing
-        ${{ if eq(parameters.TestSamples, 'true') }}:
-          Samples Linux Node 10:
-            Pool: azsdk-pool-mms-ubuntu-1804-general
-            OSVmImage: MMSUbuntu18.04
-            TestType: "sample"
-            NodeTestVersion: "10.x"            
-        
-        # Add matrix entry for min-max testing
-        ${{ if eq(parameters.TestMinMax, 'true') }}:
-          MaxVersion_Node:
-            Pool: azsdk-pool-mms-ubuntu-1804-general
-            OSVmImage: MMSUbuntu18.04
-            TestType: "node"
-            DependencyVersion: max
-            NodeTestVersion: "$(NodeVersion)"
-            TestResultsFiles: "**/test-results.xml"            
-          MinVersion_Node:
-            Pool: azsdk-pool-mms-ubuntu-1804-general
-            OSVmImage: MMSUbuntu18.04
-            TestType: "node"
-            DependencyVersion: min
-            NodeTestVersion: "$(NodeVersion)"
-            TestResultsFiles: "**/test-results.xml"                  
-
-        # Add matrix entry for same dep testing
-        ${{ if eq(parameters.TestSame, 'true') }}:
-          SameVersion_Node:
-            Pool: azsdk-pool-mms-ubuntu-1804-general
-            OSVmImage: MMSUbuntu18.04
-            TestType: "node"
-            DependencyVersion: same
-            NodeTestVersion: "$(NodeVersion)"
-            TestResultsFiles: "**/test-results.xml"   
-
-        # Add matrix entry for canary testing
-        ${{ if eq(parameters.TestCanary, 'true') }}:
-          CanaryTest_Node:
-            Pool: azsdk-pool-mms-ubuntu-1804-general
-            OSVmImage: MMSUbuntu18.04
-            TestType: "node"
-            NodeTestVersion: "$(NodeVersion)"
-            TestResultsFiles: "**/test-results.xml"
-            SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources-preview)
-            ResourceGroupLocation: ${{ parameters.ResourceGroupLocationCanary }}
+        value: '@{}'
 
     pool:
       name: $(Pool)
       vmImage: $(OSVmImage)
 
     timeoutInMinutes: ${{ parameters.TimeoutInMinutes }}
 
+    ${{ if eq(parameters.UsePlatformContainer, 'true') }}:
+      container: $[ variables['Container'] ]
+
     steps:
       - template: ../steps/common.yml
 
       - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml
         parameters:
           AgentImage: $(OSVmImage)
 
-      - ${{ if ne(parameters.ResourceServiceDirectory, '') }}:
+      - ${{ if ne(parameters.ServiceDirectory, '') }}:
         - template: /eng/common/TestResources/deploy-test-resources.yml
           parameters:
-            ServiceDirectory: "${{ parameters.ResourceServiceDirectory }}"
-            SubscriptionConfiguration: $(SubscriptionConfiguration)
-            Location: $(ResourceGroupLocation)
+            ${{ if or(parameters.Location, parameters.CloudConfig.Location) }}:
+              Location: ${{ coalesce(parameters.Location, parameters.CloudConfig.Location) }}
+            ServiceDirectory: '${{ parameters.ServiceDirectory }}'
+            SubscriptionConfiguration: ${{ parameters.CloudConfig.SubscriptionConfiguration }}
             ArmTemplateParameters: $(ArmTemplateParameters)
 
       - script: |
@@ -196,7 +93,7 @@ jobs:
           node common/scripts/install-run-rush.js build:test -t "${{parameters.PackageName}}" --verbose -p max
         displayName: "Build test assets"
         condition: and(succeededOrFailed(),eq(variables['DependencyVersion'],''))
-      
+
       - script: |
           npm install
           node index.js --artifact-name "${{parameters.PackageName}}" --version-type $(DependencyVersion) --source-dir "$(Build.SourcesDirectory)" --test-folder "test/public"
@@ -216,8 +113,7 @@ jobs:
         workingDirectory: $(PackageTestPath)
         condition: and(succeededOrFailed(),ne(variables['DependencyVersion'],''))
 
-      - ${{if ne(parameters.PreIntegrationSteps, '')}}:
-          - template: ../steps/${{parameters.PreIntegrationSteps}}.yml
+      - ${{ parameters.PreSteps }}
 
       - pwsh: |
           npm install
@@ -247,8 +143,7 @@ jobs:
           TEST_MODE: "live"
           ${{ insert }}: ${{ parameters.EnvVars }}
 
-      - ${{if ne(parameters.PostIntegrationSteps, '')}}:
-          - template: ../steps/${{parameters.PostIntegrationSteps}}.yml
+      - ${{ parameters.PostSteps }}
 
       - script: |
           node common/scripts/npm-run-project.js "${{parameters.PackageName}}" execute:samples
@@ -258,11 +153,11 @@ jobs:
           ${{ insert }}: ${{ parameters.EnvVars }}
         condition: and(succeeded(),eq(variables['TestType'],'sample'))
 
-      - ${{ if ne(parameters.ResourceServiceDirectory, '') }}:
+      - ${{ if ne(parameters.ServiceDirectory, '') }}:
         - template: /eng/common/TestResources/remove-test-resources.yml
           parameters:
-            ServiceDirectory: "${{ parameters.ResourceServiceDirectory }}"
-            SubscriptionConfiguration: $(SubscriptionConfiguration)
+            ServiceDirectory: "${{ parameters.ServiceDirectory }}"
+            SubscriptionConfiguration: '${{ parameters.CloudConfig.SubscriptionConfiguration }}'
 
       - task: PublishCodeCoverageResults@1
         displayName: "Publish NodeJs Code Coverage to DevOps"
@@ -302,17 +197,12 @@ jobs:
       - task: PublishTestResults@2
         inputs:
           searchFolder: "$(System.DefaultWorkingDirectory)/sdk"
-          testResultsFiles: "**/test-results.xml"
-          testRunTitle: "$(OSName) - NodeJS - Integration Tests - [Node $(NodeTestVersion)]"
-        condition: and(always(), eq(variables['TestType'], 'node'))
-        displayName: "Publish NodeJS integration test results"
-
-      # It's important for performance to pass "sdk" as "searchFolder" to avoid looking under root "node_modules".
-      # PublishTestResults.searchFolder only supports absolute paths, not relative.
-      - task: PublishTestResults@2
-        inputs:
-          searchFolder: "$(System.DefaultWorkingDirectory)/sdk"
-          testResultsFiles: "**/test-results.browser.xml"
-          testRunTitle: "$(OSName) - Browser - Integration Tests - [Node $(NodeTestVersion)]"
-        condition: and(always(), eq(variables['TestType'], 'browser'))
-        displayName: "Publish browser integration test results"
+          testResultsFiles: "$(TestResultsFiles)"
+          testRunTitle: "$(OSName) - $(TestType) - Integration Tests - [Node $(NodeTestVersion)]"
+        condition: >
+          and(
+            always(),
+            ne(variables['TestResultsFiles'], ''),
+            or(eq(variables['TestType'], 'node'), eq(variables['TestType'], 'browser'))
+          )
+        displayName: "Publish $(TestType) integration test results"

eng/pipelines/templates/stages/archetype-sdk-tests.yml

@@ -0,0 +1,103 @@
+parameters:
+  - name: PackageName
+    type: string
+    default: ""
+  - name: ServiceDirectory
+    type: string
+    default: ""
+  - name: EnvVars
+    type: object
+    default: {}
+  - name: MaxParallel
+    type: number
+    default: 0
+  - name: TimeoutInMinutes
+    type: number
+    default: 0
+  - name: PublishCodeCoverage
+    type: boolean
+    default: false
+  - name: Location
+    type: string
+    default: ""
+  - name: Clouds
+    type: string
+    default: 'Public'
+  - name: SupportedClouds
+    type: string
+    default: 'Public,Canary'
+  - name: UnsupportedClouds
+    type: string
+    default: ''
+  - name: PreSteps
+    type: object
+    default: []
+  - name: PostSteps
+    type: object
+    default: []
+  - name: CloudConfig
+    type: object
+    default:
+      Public:
+        SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
+      Preview:
+        SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources-preview)
+      Canary:
+        SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
+        Location: 'centraluseuap'
+      UsGov:
+        SubscriptionConfiguration: $(sub-config-gov-test-resources)
+      China:
+        SubscriptionConfiguration: $(sub-config-cn-test-resources)
+  - name: MatrixConfigs
+    type: object
+    default:
+      - Name: Js_live_test_base
+        Path: eng/pipelines/templates/stages/platform-matrix.json
+        Selection: sparse
+        GenerateVMJobs: true
+  - name: AdditionalMatrixConfigs
+    type: object
+    default: []
+  - name: MatrixFilters
+    type: object
+    default: []
+
+stages:
+- ${{ each cloud in parameters.CloudConfig }}:
+  # TODO: re-enable tests-weekly allow filter once sovereign cloud live tests are stable: https://github.com/Azure/azure-sdk/issues/2074
+  # Run all clouds by default for weekly test pipeline, except for clouds specifically unsupported by the calling pipeline
+  # - ${{ if or(contains(parameters.Clouds, cloud.key), contains(variables['Build.DefinitionName'], 'tests-weekly')) }}:
+  - ${{ if or(contains(parameters.Clouds, cloud.key), and(contains(variables['Build.DefinitionName'], 'tests-weekly'), contains(parameters.SupportedClouds, cloud.key))) }}:
+    - ${{ if not(contains(parameters.UnsupportedClouds, cloud.key)) }}:
+      - stage: ${{ cloud.key }}
+        dependsOn: []
+        jobs:
+        - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml
+          parameters:
+            JobTemplatePath: /eng/pipelines/templates/jobs/archetype-sdk-integration.yml
+            AdditionalParameters:
+              PackageName: ${{ parameters.PackageName }}
+              ServiceDirectory: ${{ parameters.ServiceDirectory }}
+              EnvVars: ${{ parameters.EnvVars }}
+              MaxParallel: ${{ parameters.MaxParallel }}
+              TimeoutInMinutes: ${{ parameters.TimeoutInMinutes }}
+              PublishCodeCoverage: ${{ parameters.PublishCodeCoverage }}
+              PreSteps:
+                - ${{ parameters.PreSteps }}
+              PostSteps:
+                - ${{ parameters.PostSteps }}
+            MatrixConfigs:
+              # Enumerate platforms and additional platforms based on supported clouds (sparse platform<-->cloud matrix).
+              - ${{ each config in parameters.MatrixConfigs }}:
+                -  ${{ config }}
+              - ${{ each config in parameters.AdditionalMatrixConfigs }}:
+                -  ${{ config }}
+            MatrixFilters:
+              - ${{ each cloudFilter in cloud.value.MatrixFilters }}:
+                - ${{ cloudFilter }}
+              - ${{ parameters.MatrixFilters }}
+            CloudConfig:
+              SubscriptionConfiguration: ${{ cloud.value.SubscriptionConfiguration }}
+              Location: ${{ coalesce(parameters.Location, cloud.value.Location) }}
+              Cloud: ${{ cloud.key }}