Onboarding v2 (Azure#31457)

* Onboarding v2

* docindex changes to test v2 onboarding

* azure-mgmt-mixedreality is properly onboarded to ToC via metadata/preview/azure-mgmt-mixedreality

* Remove the rest of the function

* Add changes in eng/common

* DocRepoLocation

* Include package source override in eng/common

* Revert eng/common changes, use intended docindex.yml

* Revert "Revert eng/common changes, use intended docindex.yml"

This reverts commit 4b37573.

* Use . accessor instead of dictionary accessor. Works well for both dictionary and pscustomobject

* Remove eng/common changes used for testing

* Remove testing changes from docindex.yml

* Remvoe comment

* Use Update-DocsMsPackages.ps1

Author: danieljurek

eng/scripts/Language-Settings.ps1

@@ -8,6 +8,7 @@ $GithubUri = "https://github.com/Azure/azure-sdk-for-python"
 $PackageRepositoryUri = "https://pypi.org/project"
 
 ."$PSScriptRoot/docs/Docs-ToC.ps1"
+."$PSScriptRoot/docs/Docs-Onboarding.ps1"
 
 function Get-AllPackageInfoFromRepo ($serviceDirectory)
 {
@@ -487,8 +488,7 @@ function UpdateDocsMsPackages($DocConfigFile, $Mode, $DocsMetadata, $PackageSour
         };
         exclude_path = @("test*","example*","sample*","doc*");
       }
-    }
-    else {
+    } else {
       $package = [ordered]@{
           package_info = [ordered]@{
             name = $packageName;
@@ -612,10 +612,15 @@ function Validate-Python-DocMsPackages ($PackageInfo, $PackageInfos, $PackageSou
   }
 
   $allSucceeded = $true
-  foreach ($package in $PackageInfos) {
+  foreach ($item in $PackageInfos) {
+    # Some packages 
+    if ($item.Version -eq 'IGNORE') { 
+      continue
+    }
+
     $result = ValidatePackage `
-      -packageName $package.Name `
-      -packageVersion $package.Version `
+      -packageName $item.Name `
+      -packageVersion "==$($item.Version)" `
       -PackageSourceOverride $PackageSourceOverride `
       -DocValidationImageId $DocValidationImageId
 

eng/scripts/docs/Docs-Onboarding.ps1

@@ -0,0 +1,62 @@
+. "$PSScriptRoot/Docs-ToC.ps1"
+
+# $SetDocsPackageOnboarding = "Set-${Language}-DocsPackageOnboarding"
+function Set-python-DocsPackageOnboarding($moniker, $metadata, $docRepoLocation, $packageSourceOverride) { 
+    $onboardingFile = GetOnboardingFileForMoniker $docRepoLocation $moniker
+
+    $onboardingSpec = Get-Content $onboardingFile -Raw | ConvertFrom-Json -AsHashtable
+
+    $packagesToOnboard = @()
+    foreach ($package in $metadata) {
+        $packageSpec = [ordered]@{
+            package_info = [ordered]@{
+                name = $package.Name
+                install_type = 'pypi'
+                prefer_source_distribution = 'true'
+                version = "==$($package.Version)"
+            }
+            exclude_path = @("test*","example*","sample*","doc*")
+        }
+
+        if ($packageSourceOverride) {
+            $packageSpec['package_info']['extra_index_url'] = $packageSourceOverride
+        }
+
+        if ($package.ContainsKey('DocsCiConfigProperties')) {
+            $overrides = $package['DocsCiConfigProperties']
+            
+            # Merge properties from package_info object (duplicate values will)
+            # be overwritten
+            if ($overrides.ContainsKey('package_info')) {
+                foreach ($key in $overrides['package_info'].Keys) {
+                    $packageSpec['package_info'][$key] = $overrides['package_info'][$key]
+                }
+            }
+
+            # Directly override other keys like exlcude_path
+            foreach ($key in $overrides.Keys) { 
+                if ($key -in @('package_info')) {
+                    # Skip over keys that have already been processed
+                    continue
+                }
+
+                $packageSpec[$key] = $overrides[$key]
+            }
+        }
+
+        $packagesToOnboard += $packageSpec
+    }
+
+    $onboardingSpec['packages'] = $packagesToOnboard
+
+    Set-Content `
+        -Path $onboardingFile `
+        -Value ($onboardingSpec | ConvertTo-Json -Depth 100)
+}
+
+# $GetDocsPackagesAlreadyOnboarded = "Get-${Language}-DocsPackagesAlreadyOnboarded"
+function Get-python-DocsPackagesAlreadyOnboarded($docRepoLocation, $moniker) {
+    return &$GetOnboardedDocsMsPackagesForMonikerFn `
+        -DocRepoLocation $docRepoLocation `
+        -moniker $moniker
+}

eng/scripts/docs/Docs-ToC.ps1

@@ -1,12 +1,19 @@
-function Get-python-OnboardedDocsMsPackagesForMoniker($DocRepoLocation, $moniker) {
-  $packageOnboardingFile = ""
-  if ("latest" -eq $moniker) {
-    $packageOnboardingFile = "$DocRepoLocation/ci-configs/packages-latest.json"
-  }
-  if ("preview" -eq $moniker) {
-    $packageOnboardingFile = "$DocRepoLocation/ci-configs/packages-preview.json"
+function GetOnboardingFileForMoniker($docRepoLocation, $moniker) { 
+  $packageOnboardingFile = 'ci-configs/packages-latest.json'
+  if ($moniker -eq 'preview') {
+    $packageOnboardingFile = 'ci-configs/packages-preview.json'
+  } elseif ($moniker -eq 'legacy') {
+    $packageOnboardingFile = 'ci-configs/packages-legacy.json'
   }
 
+  return (Join-Path $docRepoLocation $packageOnboardingFile)
+}
+
+function Get-python-OnboardedDocsMsPackagesForMoniker($DocRepoLocation, $moniker) {
+  $packageOnboardingFile = GetOnboardingFileForMoniker `
+    -docRepoLocation $DocRepoLocation `
+    -moniker $moniker
+
   $onboardedPackages = @{}
   $onboardingSpec = ConvertFrom-Json (Get-Content $packageOnboardingFile -Raw)
   foreach ($spec in $onboardingSpec.packages) {
@@ -85,23 +92,6 @@ function Get-python-DocsMsTocChildrenForManagementPackages($packageMetadata, $do
   return @($packageMetadata.Package)
 }
 
-function addManagementPackage($serviceEntry, $packageName) {
-  for ($i = 0; $i -lt $serviceEntry.items.Count; $i++) {
-    if ($serviceEntry.items[$i].name -eq "Management") {
-      $serviceEntry.items[$i].children += @($packageName)
-      return $serviceEntry
-    }
-  }
-
-  $serviceEntry.items += [PSCustomObject]@{
-    name  = "Management";
-    landingPageType = 'Service';
-    children = @($packageName)
-  }
-
-  return $serviceEntry
-}
-
 function Get-python-RepositoryLink($packageInfo) {
   return "$PackageRepositoryUri/$($packageInfo.Package)"
 }
@@ -118,12 +108,6 @@ function Get-python-UpdatedDocsMsToc($toc) {
         children = @("adal")
       }
     }
-
-    # azure-mgmt-mixedreality is not onboarded in an obvious manner, it is
-    # onboarded using a URL to a specific dist.
-    if ($services[$i].name -eq 'Mixed Reality') {
-      $services[$i] = addManagementPackage $services[$i] 'azure-mgmt-mixedreality'
-    }
   }
 
   $functionService =  [PSCustomObject]@{