Sync eng/common directory with azure-sdk-tools for PR 6632 (Azure#26916)

Sync eng/common directory with azure-sdk-tools for PR
Azure/azure-sdk-tools#6632 See [eng/common
workflow](https://github.com/Azure/azure-sdk-tools/blob/main/eng/common/README.md#workflow)

---------

Co-authored-by: Daniel Jurek <djurek@microsoft.com>

Author: azure-sdk

eng/common/scripts/Update-DocsMsPackages.ps1

@@ -37,90 +37,93 @@ param (
 )
 
 . (Join-Path $PSScriptRoot common.ps1)
+. "$PSScriptRoot/../../scripts/docs/Docs-Onboarding.ps1"
 
-function GetDocsMetadataForMoniker($moniker) {
-  $searchPath = Join-Path $DocRepoLocation 'metadata' $moniker
-  if (!(Test-Path $searchPath)) {
-    return @()
-  }
-  $paths = Get-ChildItem -Path $searchPath -Filter *.json
+Set-StrictMode -Version 3
 
+function GetMetadata($moniker) {
+  $jsonFiles = Get-ChildItem -Path (Join-Path $DocRepoLocation "metadata/$moniker") -Filter *.json
   $metadata = @()
-  foreach ($path in $paths) {
-    $fileContents = Get-Content $path -Raw
-    $fileObject = ConvertFrom-Json -InputObject $fileContents
-    $versionGa = ''
-    $versionPreview = ''
-    if ($moniker -eq 'latest') {
-      $versionGa = $fileObject.Version
-    } else {
-      $versionPreview = $fileObject.Version
-    }
-
-    $entry = @{
-      Package = $fileObject.Name;
-      VersionGA = $versionGa;
-      VersionPreview = $versionPreview;
-      RepoPath = $fileObject.ServiceDirectory;
-      Type = $fileObject.SdkType;
-      New = $fileObject.IsNewSdk;
-    }
-    if ($fileObject.PSObject.Members.Name -contains "Group")
-    {
-      $entry.Add("GroupId", $fileObject.Group)
-    }
-    $metadata += $entry
+  foreach ($jsonFile in $jsonFiles) {
+    # Converting to a hashtable gives more beneficial semantics for handling
+    # metadata (easier to check existence of property, easier to add new
+    # properties)
+    $metadata += Get-Content $jsonFile -Raw | ConvertFrom-Json -AsHashtable
   }
 
   return $metadata
 }
 
-function GetDocsMetadata() {
-  # Read metadata from docs repo
-  $metadataByPackage = @{}
-  foreach ($package in GetDocsMetadataForMoniker 'latest') {
-    if ($metadataByPackage.ContainsKey($package.Package)) {
-      LogWarning "Duplicate package in latest metadata: $($package.Package)"
-    }
-    Write-Host "Adding latest package: $($package.Package)"
-    $metadataByPackage[$package.Package] = $package
+function ValidatePackageForOnboarding2($package) {
+  if (!(Test-Path "Function:$ValidateDocsMsPackagesFn")) {
+    return $true
   }
 
-  foreach ($package in GetDocsMetadataForMoniker 'preview') {
-    if ($metadataByPackage.ContainsKey($package.Package)) {
-      # Merge VersionPreview of each object
-      Write-Host "Merging preview package version for $($package.Package))"
-      $metadataByPackage[$package.Package].VersionPreview = $package.VersionPreview
-    } else {
-      Write-Host "Adding preview package: $($package.Package)"
-      $metadataByPackage[$package.Package] = $package
-    }
-  }
-
-  # TODO - Add a call to GetDocsMetadataForMoniker for 'legacy' when that is implemented
-
-  return $metadataByPackage.Values
+  return &$ValidateDocsMsPackagesFn `
+    -PackageInfo $package `
+    -DocValidationImageId $ImageId `
+    -DocRepoLocation $DocRepoLocation
 }
 
-if ($UpdateDocsMsPackagesFn -and (Test-Path "Function:$UpdateDocsMsPackagesFn")) {
-
+$MONIKERS = @('latest', 'preview', 'legacy')
+foreach ($moniker in $MONIKERS) {
   try {
-    $docsMetadata = GetDocsMetadata
-    &$UpdateDocsMsPackagesFn -DocsRepoLocation $DocRepoLocation -DocsMetadata $docsMetadata -PackageSourceOverride $PackageSourceOverride -DocValidationImageId $ImageId
-  } catch {
-    LogError "Exception while updating docs.ms packages"
-    LogError $_
-    LogError $_.ScriptStackTrace
+    Write-Host "Onboarding packages for moniker: $moniker"
+    $metadata = GetMetadata $moniker
+    $alreadyOnboardedPackages = &$GetDocsPackagesAlreadyOnboarded $DocRepoLocation $moniker
+
+    # Sort the metadata entries by package name so that the output is
+    # deterministic (more simple diffs)
+    $sortedMetadata = $metadata | Sort-Object -Property '_DocsOnboardingOrdinal', 'Name'
+
+    $outputPackages = @()
+    foreach ($package in $sortedMetadata) {
+      $packageIdentity = $package.Name
+      if (Test-Path "Function:$GetPackageIdentity") {
+        $packageIdentity = &$GetPackageIdentity $package
+      }
+
+      if (!($alreadyOnboardedPackages.ContainsKey($packageIdentity))) {
+        Write-Host "Evaluating package for onboarding: $($packageIdentity)"
+        if ($package.ContainsKey('_SkipDocsValidation') -and $true -eq $package['_SkipDocsValidation']) {
+          Write-Host "Skip validation for package: $($packageIdentity)"
+        }
+        elseif (!(ValidatePackageForOnboarding2 $package)) {
+          LogWarning "Skip adding package that did not pass validation: $($packageIdentity)"
+          continue
+        }
+
+        Write-Host "Add new package: $($packageIdentity)@$($package.Version)"
+        $outputPackages += $package
+        continue
+      }
+
+      $oldPackage = $alreadyOnboardedPackages[$packageIdentity]
+
+      if ($oldPackage.Version -ne $package.Version) {
+        if (!(ValidatePackageForOnboarding2 $package)) {
+          LogWarning "Omitting package that failed validation: $($packageIdentity)@$($package.Version)"
+          continue
+        }
+
+        Write-Host "Update package: $($packageIdentity)@$($oldPackage.Version) to $($packageIdentity)@$($package.Version)"
+        $outputPackages += $package
+        continue
+      }
+
+      Write-Host "Unchanged package: $($packageIdentity)@$($package.Version)"
+      $outputPackages += $package
+    }
+
+    &$SetDocsPackageOnboarding $moniker $outputPackages $DocRepoLocation $PackageSourceOverride
+  }
+  catch {
+    Write-Host "Error onboarding packages for moniker: $moniker"
+    Write-Host "Error: $_"
+    Write-Host "Stacktrace: $($_.ScriptStackTrace)"
+    Write-Error $_
     exit 1
   }
-
-} else {
-  LogError "The function for '$UpdateFn' was not found.`
-  Make sure it is present in eng/scripts/Language-Settings.ps1 and referenced in eng/common/scripts/common.ps1.`
-  See https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/common_engsys.md#code-structure"
-  exit 1
 }
 
-# Exit 0 so DevOps doesn't fail the build when the last command called by the
-# domain-specific function exited with a non-zero exit code.
 exit 0

eng/common/scripts/common.ps1

@@ -40,6 +40,7 @@ if (!(Get-Variable -Name "LanguageDisplayName" -ValueOnly -ErrorAction "Ignore")
 }
 
 # Transformed Functions
+# Expected to be set in eng/scripts/Language-Settings.ps1
 $GetPackageInfoFromRepoFn = "Get-${Language}-PackageInfoFromRepo"
 $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile"
 $PublishGithubIODocsFn = "Publish-${Language}-GithubIODocs"
@@ -60,3 +61,8 @@ $GetRepositoryLinkFn = "Get-${Language}-RepositoryLink"
 $GetEmitterAdditionalOptionsFn = "Get-${Language}-EmitterAdditionalOptions"
 $GetEmitterNameFn = "Get-${Language}-EmitterName"
 $GetEmitterPackageJsonPathFn = "Get-${Language}-EmitterPackageJsonPath"
+
+# Expected to be set in eng/scripts/docs/Docs-Onboarding.ps1
+$SetDocsPackageOnboarding = "Set-${Language}-DocsPackageOnboarding"
+$GetDocsPackagesAlreadyOnboarded = "Get-${Language}-DocsPackagesAlreadyOnboarded"
+$GetPackageIdentity = "Get-${Language}-PackageIdentity"