Bug fix and improvement to aggregate reports pipeline (#25503)

* Bug fix and improvement to aggregate reports pipeline

* Build all packages

* Add script for download packages from artifacts dev feed

* Update dependency graph

* Use alpha versions when downloading nuget files from DevOps feed

* Use alpha versions where available, else use latest dev version

* Switch to using Install-Package

Author: chidozieononiwu

README.md

@@ -1,6 +1,6 @@
 # Azure SDK for .NET
 
-[![Packages](https://img.shields.io/badge/packages-latest-blue.svg)](https://azure.github.io/azure-sdk/releases/latest/dotnet.html) [![Dependencies](https://img.shields.io/badge/dependency-report-blue.svg)](https://azuresdkartifacts.blob.core.windows.net/azure-sdk-for-net/dependencies/dependencies.html) [![Dependencies Graph](https://img.shields.io/badge/dependency-graph-blue.svg)](https://azuresdkartifacts.blob.core.windows.net/azure-sdk-for-net/dependencies/InterdependencyGraph.html)
+[![Packages](https://img.shields.io/badge/packages-latest-blue.svg)](https://azure.github.io/azure-sdk/releases/latest/dotnet.html) [![Dependencies](https://img.shields.io/badge/dependency-report-blue.svg)](https://azuresdkartifacts.blob.core.windows.net/azure-sdk-for-net/dependencies/dependencies.html) [![Dependencies Graph](https://img.shields.io/badge/dependency-graph-blue.svg)](https://azuresdkartifacts.blob.core.windows.net/azure-sdk-for-net/dependencies/dependencyGraph/dagre.html)
 
 This repository is for active development of the Azure SDK for .NET. For consumers of the SDK we recommend visiting our [public developer docs](https://docs.microsoft.com/dotnet/azure/) or our versioned [developer docs](https://azure.github.io/azure-sdk-for-net).
 

doc/Data Plane Code Generation/AzureSDKCodeGeneration_DataPlane_Quickstart.md

@@ -84,7 +84,7 @@ In Visual Studio:
 
 You'll notice that the `Azure.<group>.<service>` project has `properties/AssemblyInfo.cs` file in it. Replace `Microsoft.Test` namespace with the correct resource provider namepace for your service.
 
-**Learn more:** see the [Azure Services Resource Providers](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/azure-services-resource-providers) for the list of possible namespaces.
+**Learn more:** see the [Azure Services Resource Providers](https://docs.microsoft.com/azure/azure-resource-manager/management/azure-services-resource-providers) for the list of possible namespaces.
 
 #### 5. Save and commit your changes to your branch
 

eng/pipelines/aggregate-reports.yml

@@ -1,12 +1,3 @@
-resources:
-  pipelines:
-  - pipeline: net-core
-    project: internal
-    source: net - core
-    branch: main
-    tags: 
-    - scheduled
-
 trigger: none
 
 pr:
@@ -31,10 +22,16 @@ stages:
       - job: GenerateReports
         timeoutInMinutes: 120
         steps:
-          - download: net-core
-            artifact: packages
-            patterns: '*'
-            displayName: "Download packages artifacts from 'net - core' pipeline "
+          - template: /eng/pipelines/templates/steps/install-dotnet.yml
+
+          - task: PowerShell@2
+            displayName: Download Latest Dev Packages
+            inputs:
+              pwsh: true
+              filePath: 'eng/scripts/DownloadDevPackages.ps1'
+              arguments: >
+                -WorkingDirectory '$(Build.ArtifactStagingDirectory)'
+                -NupkgFilesDestination 'nupkgFiles'
 
           - pwsh: |
               npx cspell lint `
@@ -53,39 +50,27 @@ stages:
               CheckLinkGuidance: $true
 
           - pwsh: |
-              mkdir "$(System.ArtifactsDirectory)/BuildArtifacts"
-              ls "$(PIPELINE.WORKSPACE)/net-core/packages"
-              Copy-Item -Path "$(PIPELINE.WORKSPACE)/net-core/packages/*" -Destination "$(System.ArtifactsDirectory)/BuildArtifacts"
-            displayName: Create Artifact Directory
-
-          - pwsh: |
-              mkdir '$(System.ArtifactsDirectory)/Packages'
-              Move-Item -Path '$(System.ArtifactsDirectory)/BuildArtifacts/*' -Destination '$(System.ArtifactsDirectory)/Packages/' -Include Azure.*.nupkg -Exclude *.symbols.nupkg
-            displayName: Isolate packages to process
-
-          - pwsh: |
-              mkdir '$(Build.ArtifactStagingDirectory)/reports'
-              Copy-Item -Path '$(Build.SourcesDirectory)/eng/common/InterdependencyGraph.html' -Destination '$(Build.ArtifactStagingDirectory)/reports/InterdependencyGraph.html'
-            displayName: Setup reports directory
+              New-Item -Path '$(Build.ArtifactStagingDirectory)' -Name 'reports' -ItemType 'directory'
+            displayName: Create Directory for Aggregate Reports
 
           - task: PowerShell@2
             displayName: Generate Dependency Report
             inputs:
               pwsh: true
               filePath: 'eng/scripts/dependencies/AnalyzeDeps.ps1'
               arguments: >
-                -PackagesPath '$(System.ArtifactsDirectory)/Packages/'
+                -PackagesPath '$(Build.ArtifactStagingDirectory)/nupkgFiles'
                 -LockfilePath '$(Build.SourcesDirectory)/eng/Packages.Data.props'
                 -OutPath '$(Build.ArtifactStagingDirectory)/reports/dependencies.html'
-                -DumpPath '$(Build.ArtifactStagingDirectory)/reports/data.js'
+                -DumpPath '$(Build.ArtifactStagingDirectory)/reports'
 
           - task: PowerShell@2
             displayName: 'Generate azure-sdk.deps.json'
             inputs:
               pwsh: true
               filePath: 'eng/scripts/dependencies/generate-deps.ps1'
               arguments: >
-                -PackagesPath '$(System.ArtifactsDirectory)/Packages/'
+                -PackagesPath '$(Build.ArtifactStagingDirectory)/nupkgFiles'
                 -DepsOutputFile '$(Build.ArtifactStagingDirectory)/reports/azure-sdk.deps.json'
                 -ProjectRefPath '$(Build.ArtifactStagingDirectory)/reports'
 
@@ -122,15 +107,27 @@ stages:
               artifactName: reports
               path: '$(Build.ArtifactStagingDirectory)/reports'
 
-          - task: AzureFileCopy@2
-            displayName: 'Upload dependency report'
+          - task: AzureFileCopy@4
+            displayName: 'Upload Dependency Report'
             inputs:
-              sourcePath: '$(Build.ArtifactStagingDirectory)/reports'
+              sourcePath: '$(Build.ArtifactStagingDirectory)/reports/*'
               azureSubscription: 'Azure SDK Artifacts'
               destination: AzureBlob
               storage: azuresdkartifacts
               containerName: 'azure-sdk-for-net'
               blobPrefix: dependencies
+              AdditionalArgumentsForBlobCopy: '--exclude-pattern=*data.js*'
+
+          - task: AzureFileCopy@4
+            displayName: 'Upload Dependency Graph'
+            inputs:
+              sourcePath: '$(Build.ArtifactStagingDirectory)/reports/*'
+              azureSubscription: 'Azure SDK Artifacts'
+              destination: AzureBlob
+              storage: azuresdkartifacts
+              containerName: 'azure-sdk-for-net'
+              blobPrefix: dependencies/dependencyGraph
+              AdditionalArgumentsForBlobCopy: '--include-pattern=*data.js*'
 
           - task: PowerShell@2
             displayName: "Verify Repository Resource Refs"

eng/scripts/DownloadDevPackages.ps1

@@ -0,0 +1,69 @@
+param (
+    [Parameter(Mandatory=$True)]
+    [string] $WorkingDirectory,
+    [Parameter(Mandatory=$True)]
+    [string] $NupkgFilesDestination,
+    # Install-Package requires a v2 nuget feed.
+    [string] $NugetSource="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v2",
+    [string] $FeedId="azure-sdk-for-net"
+)
+
+. (Join-Path $PSScriptRoot ".." common scripts common.ps1)
+
+$allPackages = Get-AllPkgProperties
+$trackTwoPackages = $allPackages.Where({ $_.IsNewSdk })
+
+Write-Host "Number of track two packages $($trackTwoPackages.Count)"
+
+Push-Location $WorkingDirectory
+$nugetPackagesPath = Join-Path $WorkingDirectory nugetPackages
+New-Item -Path $WorkingDirectory -Type "directory" -Name "nugetPackages"
+
+
+foreach ($package in $trackTwoPackages)
+{
+  $packageVersion = [AzureEngSemanticVersion]::ParseVersionString($package.Version)
+  $packageVersion.IsPreRelease = $false # Clear prerelease so ToString strips it prerelease label.
+  $packageVersionBase = $packageVersion.ToString()
+
+  # To workaround some older invalid packages (i.e. Iot->IoT renamed packages) start the alpha version range to start at 6/21
+  $installedPackage = Install-Package -Name $package.Name `
+    -Source $NugetSource `
+    -AllowPrereleaseVersions `
+    -MinimumVersion "$packageVersionBase-alpha.202106" `
+    -MaximumVersion "$packageVersionBase-alphab" `
+    -Destination $nugetPackagesPath `
+    -Force `
+    -SkipDependencies `
+    -ErrorAction Ignore
+
+  if ($installedPackage)
+  {
+    Write-Host "Installed $($installedPackage.Name) $($installedPackage.Version)"
+  }
+  else 
+  {
+    # Install the latest available version if no alpha version is found
+    $latestInstalledPackage  = Install-Package -Name $package.Name `
+      -Source $NugetSource `
+      -AllowPrereleaseVersions `
+      -Destination $nugetPackagesPath `
+      -Force `
+      -SkipDependencies `
+      -ErrorAction Ignore
+
+    if ($latestInstalledPackage)
+    {
+      Write-Host "Installed latest version $($latestInstalledPackage.Name) $($latestInstalledPackage.Version)"
+    }
+    else
+    {
+      Write-Host "Did not find any matching package $($package.Name)"
+    }
+  }
+}
+
+$nupkgDirPath = Join-Path $WorkingDirectory $NupkgFilesDestination
+New-Item -Path $WorkingDirectory -Type "directory" -Name $NupkgFilesDestination
+
+Get-ChildItem -Path $nugetPackagesPath -Include *.nupkg -Recurse | Copy-Item -Destination $nupkgDirPath

eng/scripts/dependencies/AnalyzeDeps.ps1

@@ -249,7 +249,11 @@ if ($DumpPath) {
   Write-Host "Generating JSONP data export..."
   $Internal = $Pkgs.Keys | ForEach-Object ToString
   $DumpData = Get-PackageExport $Pkgs $Internal
-  "const data = " + (ConvertTo-Json -InputObject $DumpData -Compress -Depth 10) + ";" | Out-File -FilePath $DumpPath
+  Write-Host $DumpData
+  $DumpDataJson = ConvertTo-Json -InputObject $DumpData -Compress -Depth 10
+  Write-Host $DumpDataJson
+  $DumpDataJson | Out-File -FilePath "${DumpPath}/arcdata.json"
+  "const data = " + $DumpDataJson + ";" | Out-File -FilePath "${DumpPath}/data.js"
 }
 
 exit $ExitCode

eng/scripts/dependencies/generate-deps.ps1

@@ -7,6 +7,14 @@ param(
 mkdir $ProjectRefPath -force | out-null
 $projRefsPath = Join-Path $ProjectRefPath "azure-sdk-projectreferences.props"
 
+#See issues
+# https://github.com/Azure/azure-sdk-for-net/issues/25827
+# https://github.com/Azure/azure-sdk-for-net/issues/25824
+$ignoreList = @(
+  "Azure.Monitor.OpenTelemetry.Exporter",
+  "Microsoft.Azure.Messaging.EventGrid.CloudNativeCloudEvents"
+)
+
 $projectRefsContents = @"
 <?xml version="1.0" encoding="utf-8"?>
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
@@ -18,7 +26,10 @@ foreach ($pkg in (Get-ChildItem "$PackagesPath/*" -Include *.nupkg -Exclude *.sy
 {
   if ($pkg.Name -match "(?<pkg>[^\\]+)\.(?<ver>\d+\.\d+\.\d+.*)\.nupkg") 
   {
-    $projectRefsContents += "    <PackageReference Include=`"$($matches['pkg'])`" Version=`"$($matches['ver'])`" />`n"
+    if ($matches['pkg'] -notin $ignoreList)
+    {
+      $projectRefsContents += "    <PackageReference Include=`"$($matches['pkg'])`" Version=`"$($matches['ver'])`" />`n"
+    }
   }
 }
 
@@ -29,6 +40,6 @@ $projectRefsContents += @"
 
 Set-Content -Path $projRefsPath -Value $projectRefsContents
 
-$cmd = "dotnet build $PSScriptRoot/azure-sdk.deps.csproj /t:GenerateDepsFile /p:PublishDepsFilePath=$DepsOutputFile /p:AzureSdkProjectReferencesPath=$projRefsPath /p:RestoreAdditionalProjectSources=$PackagesPath"
+$cmd = "dotnet build $PSScriptRoot/azure-sdk.deps.csproj /t:GenerateDepsFile /p:PublishDepsFilePath=""$DepsOutputFile"" /p:AzureSdkProjectReferencesPath=""$projRefsPath"" /p:RestoreAdditionalProjectSources=""$PackagesPath"""
 Write-Host $cmd
 Invoke-Expression $cmd

sdk/communication/Azure.Communication.PhoneNumbers/README.md

@@ -158,7 +158,7 @@ This project has adopted the [Microsoft Open Source Code of Conduct][coc]. For m
 [coc_faq]: https://opensource.microsoft.com/codeofconduct/faq/
 [coc_contact]: mailto:opencode@microsoft.com
 <!--[package]: https://www.nuget.org/packages/Azure.Communication.PhoneNumbers-->
-[phone_numbers]: https://docs.microsoft.com/azure/communication-services/quickstarts/telephony-sms/get-phone-number?pivots=programming-language-csharp
+[phone_numbers]: https://docs.microsoft.com/azure/communication-services/quickstarts/telephony/get-phone-number?pivots=platform-azp
 [product_docs]: https://docs.microsoft.com/azure/communication-services/overview
 [nuget]: https://www.nuget.org/
 [communication_resource_docs]: https://docs.microsoft.com/azure/communication-services/quickstarts/create-communication-resource?tabs=windows&pivots=platform-azp