Eventhubs/pipeline pivot (Azure#17804)

* [Event Hubs] Local Pipeline Exclusions

The focus of these changes is to revisit the approach of extending the
central pipeline for uniform project exclusions and, instead, pivot to
using a local services project specific to Event Hubs.

* Add SDKType support to enable filtering

Plumb SDKType support through our pipeline templates
to allow support for filtering based on the type of SDK
we want to build in our pipelines.

Author: weshaggard

eng/pipelines/templates/jobs/archetype-sdk-client.yml

@@ -8,6 +8,9 @@ parameters:
 - name: ArtifactName
   type: string
   default: packages
+- name: SDKType
+  type: string
+  default: client
 - name: ServiceDirectory
   type: string
   default: not-specified
@@ -23,9 +26,6 @@ parameters:
 - name: LinuxPool
   type: string
   default: azsdk-pool-mms-ubuntu-1804-general
-- name: ExcludeProjects
-  type: object
-  default: []
 
 jobs:
   - job: Build
@@ -66,9 +66,9 @@ jobs:
       - template: /eng/pipelines/templates/steps/install-dotnet.yml
       - script: >-
           dotnet pack eng/service.proj -warnaserror
+          /p:SDKType=${{ parameters.SDKType }}
           /p:ServiceDirectory=${{ parameters.ServiceDirectory }}
           /p:IncludeTests=false
-          /p:ExcludeProjects="${{ join('|', parameters.ExcludeProjects) }}"
           /p:PublicSign=false $(VersioningProperties)
           /p:Configuration=$(BuildConfiguration)
           /p:CommitSHA=$(Build.SourceVersion)
@@ -134,8 +134,8 @@ jobs:
         inputs:
           filePath: "eng/scripts/CodeChecks.ps1"
           arguments: >
-            -ServiceDirectory ${{parameters.ServiceToTest}} 
-            -ExcludeProjects "${{ join('|', parameters.ExcludeProjects) }}"
+           -ServiceDirectory ${{ parameters.ServiceToTest }}
+           -SDKType ${{ parameters.SDKType }}
           pwsh: true
           failOnStderr:  false
       - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
@@ -195,11 +195,11 @@ jobs:
       - script: >-
           dotnet test eng/service.proj --filter "(TestCategory!=Manually) & (TestCategory!=Live)" --framework $(TestTargetFramework)
           --logger "trx;LogFileName=$(TestTargetFramework).trx" --logger:"console;verbosity=normal"
-          /p:ServiceDirectory=${{parameters.ServiceToTest}}
+          /p:SDKType=${{ parameters.SDKType }}
+          /p:ServiceDirectory=${{ parameters.ServiceToTest }}
           /p:IncludeSrc=false /p:IncludeSamples=false /p:IncludePerf=false /p:IncludeStress=false
-          /p:ExcludeProjects="${{ join('|', parameters.ExcludeProjects) }}"
           /p:Configuration=$(BuildConfiguration) $(ConvertToProjectReferenceOption)
-          /p:CollectCoverage=$(CollectCoverage) /p:CodeCoverageDirectory=${{parameters.ServiceDirectory}}
+          /p:CollectCoverage=$(CollectCoverage) /p:CodeCoverageDirectory=${{ parameters.ServiceDirectory }}
         displayName: "Build & Test ($(TestTargetFramework))"
         env:
           DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1

eng/pipelines/templates/jobs/archetype-sdk-tests-jobs.yml

@@ -20,12 +20,12 @@ parameters:
 - name: Location
   type: string
   default: ''
+- name: SDKType
+  type: string
+  default: client
 - name: ServiceDirectory
   type: string
   default: not-specified
-- name: ExcludeProjects
-  type: object
-  default: []
 - name: TestSetupSteps
   type: stepList
   default: []
@@ -95,10 +95,10 @@ jobs:
           --filter "TestCategory!=Manually"
           --logger "trx"
           --logger:"console;verbosity=normal"
+          /p:SDKType=${{ parameters.SDKType }}
           /p:ServiceDirectory=${{ parameters.ServiceDirectory }}
           /p:IncludeSrc=false /p:IncludeSamples=false /p:IncludePerf=false /p:IncludeStress=false
           /p:BuildInParallel=${{ parameters.BuildInParallel }}
-          /p:ExcludeProjects="${{ join('|', parameters.ExcludeProjects) }}"
           $(AdditionalTestArguments)
 
         displayName: "Build & Test (all tests for $(TestTargetFramework))"

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -15,12 +15,12 @@ parameters:
 - name: ArtifactName
   type: string
   default: packages
+- name: SDKType
+  type: string
+  default: client
 - name: ServiceDirectory
   type: string
   default: not-specified
-- name: ExcludeProjects
-  type: object
-  default: []
 - name: ServiceToTest
   type: string
   default: ''
@@ -45,9 +45,9 @@ stages:
     jobs:
     - template: ../jobs/archetype-sdk-client.yml
       parameters:
+        SDKType: ${{ parameters.SDKType }}
         ServiceToTest: ${{ coalesce(parameters.ServiceToTest, parameters.ServiceDirectory) }}
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
-        ExcludeProjects: ${{ parameters.ExcludeProjects }}
         Artifacts: ${{ parameters.Artifacts }}
         TestPipeline: ${{ parameters.TestPipeline }}
         ArtifactName: packages
@@ -59,6 +59,7 @@ stages:
   - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}:
     - template: archetype-net-release.yml
       parameters:
+        SDKType: ${{ parameters.SDKType }}
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
         DependsOn: Build
         Artifacts: ${{ parameters.Artifacts }}

eng/pipelines/templates/stages/archetype-sdk-tests.yml

@@ -29,12 +29,12 @@ parameters:
 - name: Location
   type: string
   default: ''
+- name: SDKType
+  type: string
+  default: client
 - name: ServiceDirectory
   type: string
   default: not-specified
-- name: ExcludeProjects
-  type: object
-  default: []
 - name: TestSetupSteps
   type: stepList
   default: []
@@ -133,8 +133,8 @@ stages:
               BuildInParallel: ${{ parameters.BuildInParallel }}
               TimeoutInMinutes: ${{ parameters.TimeoutInMinutes }}
               Location: ${{ parameters.Location }}
+              SDKType: ${{ parameters.SDKType }}
               ServiceDirectory: ${{ parameters.ServiceDirectory }}
-              ExcludeProjects: ${{ parameters.ExcludeProjects }}
               TestSetupSteps: ${{ parameters.TestSetupSteps }}
             Platforms:
               # Enumerate platforms and additional platforms based on supported clouds (sparse platform<-->cloud matrix).

eng/scripts/CodeChecks.ps1

@@ -9,7 +9,7 @@ param (
     [string] $ProjectDirectory,
 
     [Parameter()]
-    [string] $ExcludeProjects
+    [string] $SDKType = "client"
 )
 
 $ErrorActionPreference = 'Stop'
@@ -76,7 +76,7 @@ try {
 
         Write-Host "Re-generating clients"
         Invoke-Block {
-            & dotnet msbuild $PSScriptRoot\..\service.proj /restore /t:GenerateCode /p:ServiceDirectory=$ServiceDirectory /p:ExcludeProjects="$ExcludeProjects"
+            & dotnet msbuild $PSScriptRoot\..\service.proj /restore /t:GenerateCode /p:SDKType=$SDKType /p:ServiceDirectory=$ServiceDirectory
 
             # https://github.com/Azure/azure-sdk-for-net/issues/8584
             # & $repoRoot\storage\generate.ps1
@@ -90,7 +90,7 @@ try {
 
     Write-Host "Re-generating listings"
     Invoke-Block {
-        & $PSScriptRoot\Export-API.ps1 -ServiceDirectory $ServiceDirectory -ExcludeProjects "$ExcludeProjects"
+        & $PSScriptRoot\Export-API.ps1 -ServiceDirectory $ServiceDirectory -SDKType $SDKType
     }
 
     if (-not $ProjectDirectory)

eng/scripts/Export-API.ps1

@@ -3,9 +3,9 @@ param (
     [Parameter(Position=0)]
     [ValidateNotNullOrEmpty()]
     [string] $ServiceDirectory,
-    [string] $ExcludeProjects
+    [string] $SDKType = "client"
 )
 
 $servicesProj = Resolve-Path "$PSScriptRoot/../service.proj"
 
-dotnet build /p:GenerateApiListingOnBuild=true /p:RunApiCompat=false /p:GeneratePackageOnBuild=false /p:Configuration=Release /p:IncludeSamples=false /p:IncludePerf=false /p:IncludeStress=false /p:IncludeTests=false /p:Scope="$ServiceDirectory" /p:ExcludeProjects="$ExcludeProjects" /restore $servicesProj
+dotnet build /p:GenerateApiListingOnBuild=true /p:RunApiCompat=false /p:GeneratePackageOnBuild=false /p:Configuration=Release /p:IncludeSamples=false /p:IncludePerf=false /p:IncludeStress=false /p:IncludeTests=false /p:Scope="$ServiceDirectory" /p:SDKType=$SDKType /restore $servicesProj

eng/service.proj

@@ -9,7 +9,6 @@
     <IncludeStress Condition="'$(IncludeStress)' == ''">true</IncludeStress>
     <IncludeSamplesApplications Condition="'$(IncludeSamplesApplications)' == ''">true</IncludeSamplesApplications>
     <IncludeSamplesApplications Condition="'$(ServiceDirectory)' != '*' or '$(IncludeSamples)' == 'false'">false</IncludeSamplesApplications>
-    <ExcludeProjectsDelimiter Condition="'$(ExcludeProjectsDelimiter)' == ''">|</ExcludeProjectsDelimiter>
     <TraversalGlobalProperties>
       CodeCoverageDirectory=$([System.IO.Path]::GetFullPath("$(CodeCoverageDirectory)", "$(MSBuildThisFileDirectory)..\sdk"));
     </TraversalGlobalProperties>
@@ -31,15 +30,6 @@
     <ProjectReference Include="@(SrcProjects)" Exclude="@(MgmtExcludePaths)" Condition="'$(IncludeSrc)' == 'true'" />
   </ItemGroup>
 
-  <!--
-     Override project inclusions and force removal of those that have been explicitly excluded by the associated parameter.
-     Assume each item identifies a project name under the service directory and build the appropriate path mask.
-  -->
-  <ItemGroup Condition="'$(ExcludeProjects)' != ''">
-    <_ExcludePathsSource Include="$(ExcludeProjects.Split('$(ExcludeProjectsDelimiter)'))" />
-    <ProjectReference Remove="@(_ExcludePathsSource -> '$(MSBuildThisFileDirectory)..\sdk\$(ServiceDirectory)\%(Identity)\**\*.csproj')" />
-  </ItemGroup>
-
   <Import Project="..\sdk\$(ServiceDirectory)\*.projects" />
 
   <Target Name="CheckProjects" AfterTargets="Build">

sdk/eventhub/ci.data.yml

@@ -0,0 +1,42 @@
+# NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file.
+
+trigger:
+  branches:
+    include:
+    - master
+    - hotfix/*
+    - release/*
+  paths:
+    include:
+    - sdk/eventhub/ci.data.yml
+    - sdk/eventhub/Microsoft.Azure.EventHubs
+    - sdk/eventhub/Microsoft.Azure.EventHubs.Processor
+    - sdk/eventhub/Microsoft.Azure.EventHubs.ServiceFabricProcessor
+
+pr:
+  branches:
+    include:
+    - master
+    - feature/*
+    - hotfix/*
+    - release/*
+  paths:
+    include:
+    - sdk/eventhub/ci.data.yml
+    - sdk/eventhub/Microsoft.Azure.EventHubs
+    - sdk/eventhub/Microsoft.Azure.EventHubs.Processor
+    - sdk/eventhub/Microsoft.Azure.EventHubs.ServiceFabricProcessor
+
+extends:
+  template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml
+  parameters:
+    SdkType: data
+    ServiceDirectory: eventhub
+    ArtifactName: packages
+    Artifacts:
+    - name: Microsoft.Azure.EventHubs
+      safeName: MicrosoftAzureEventHubs
+    - name: Microsoft.Azure.EventHubs.Processor
+      safeName: MicrosoftAzureEventHubsProcessor
+    - name: Microsoft.Azure.EventHubs.ServiceFabricProcessor
+      safeName: MicrosoftAzureEventHubsServiceFabricProcessor

sdk/eventhub/ci.legacy.yml

@@ -0,0 +1,35 @@
+# NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file.
+
+trigger:
+  branches:
+    include:
+    - master
+    - hotfix/*
+    - release/*
+  paths:
+    include:
+    - sdk/eventhub/ci.mgmt.yml
+    - sdk/eventhub/Azure.ResourceManager.EventHubs
+
+pr:
+  branches:
+    include:
+    - master
+    - feature/*
+    - hotfix/*
+    - release/*
+  paths:
+    include:
+    - sdk/eventhub/ci.mgmt.yml
+    - sdk/eventhub/Azure.ResourceManager.EventHubs
+
+extends:
+  template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml
+  parameters:
+    SdkType: mgmtclient
+    ServiceDirectory: eventhub
+    ArtifactName: packages
+    Artifacts:
+    - name: Azure.ResourceManager.EventHubs
+      safeName: AzureResourceManagerEventHubs
+