update unified pipeline script to support E2E flow (Azure#33005)

chunyu3 · web-flow · commit 7b5d1a94fe9f · 2022-12-16T16:15:45.000+08:00
* update ci script

* support local swagger

* update script

* remove unused code

* update sdk generation pipeline

* remove duplicate ymal install
diff --git a/eng/scripts/Cadl-Project-Generate.ps1 b/eng/scripts/Cadl-Project-Generate.ps1
@@ -42,6 +42,15 @@ $npmWorkingDir = Resolve-Path $tempFolder/$innerFolder
 $mainCadlFile = If (Test-Path "$npmWorkingDir/client.cadl") { Resolve-Path "$npmWorkingDir/client.cadl" } Else { Resolve-Path "$npmWorkingDir/main.cadl"}
 $resolvedProjectDirectory = Resolve-Path "$ProjectDirectory/src"
 
+if (Test-Path "$npmWorkingDir/cadl-project.yaml") {
+    $cadlProjectYaml = Get-Content -Path "$npmWorkingDir/cadl-project.yaml" -Raw
+    $yml = ConvertFrom-YAML $cadlProjectYaml
+    if ( $yml["emitters"]["@azure-tools/cadl-csharp"]["sdk-folder"]) {
+        $yml["emitters"]["@azure-tools/cadl-csharp"]["sdk-folder"] = ""
+    }
+    (ConvertTo-Yaml $yml) | Out-File "$npmWorkingDir/cadl-project.yaml"
+}
+
 try {
     Push-Location $npmWorkingDir
     NpmInstallForProject $npmWorkingDir
diff --git a/eng/scripts/Cadl-Project-Sync.ps1 b/eng/scripts/Cadl-Project-Sync.ps1
@@ -93,24 +93,29 @@ $configuration = Get-Content -Path $cadlConfigurationFile -Raw | ConvertFrom-Yam
 $pieces = $cadlConfigurationFile.Path.Replace("\","/").Split("/")
 $projectName = $pieces[$pieces.Count - 3]
 
-$specCloneDir = GetSpecCloneDir $projectName
-
-$gitRemoteValue = GetGitRemoteValue $configuration["repo"]
-
-Write-Host "Setting up sparse clone for $projectName at $specCloneDir"
 $specSubDirectory = $configuration["directory"]
-Push-Location $specCloneDir.Path
-try {
-    if (!(Test-Path ".git")) {
-        InitializeSparseGitClone $gitRemoteValue
-        UpdateSparseCheckoutFile $specSubDirectory $configuration["additionalDirectories"]
+if ( $configuration["repo"] -and $configuration["commit"]) {
+    $specCloneDir = GetSpecCloneDir $projectName
+    $gitRemoteValue = GetGitRemoteValue $configuration["repo"]
+
+    Write-Host "Setting up sparse clone for $projectName at $specCloneDir"
+    
+    Push-Location $specCloneDir.Path
+    try {
+        if (!(Test-Path ".git")) {
+            InitializeSparseGitClone $gitRemoteValue
+            UpdateSparseCheckoutFile $specSubDirectory $configuration["additionalDirectories"]
+        }
+        git checkout $configuration["commit"]
     }
-    git checkout $configuration["commit"]
-}
-finally {
-    Pop-Location
+    finally {
+        Pop-Location
+    }
+} elseif ( $configuration["spec-root-dir"] ) {
+    $specCloneDir = $configuration["spec-root-dir"]
 }
 
+
 $tempCadlDir = "$ProjectDirectory/TempCadlFiles"
 New-Item $tempCadlDir -Type Directory -Force | Out-Null
 CopySpecToProjectIfNeeded `
diff --git a/eng/scripts/Invoke-GenerateAndBuildV2.ps1 b/eng/scripts/Invoke-GenerateAndBuildV2.ps1
@@ -115,8 +115,6 @@ if ($relatedCadlProjectFolder) {
         $cadlFolder = Resolve-Path (Join-Path $swaggerDir $cadlRelativeFolder)
         $newPackageOutput = "newPackageOutput.json"
 
-        Push-Location $cadlFolder
-        trap {Pop-Location}
         $cadlProjectYaml = Get-Content -Path (Join-Path "$cadlFolder" "cadl-project.yaml") -Raw
 
         Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
@@ -131,16 +129,21 @@ if ($relatedCadlProjectFolder) {
             $service = $directories[-3];
             $namespace = $directories[-2];
         }
-        New-CADLPackageFolder -service $service -namespace $namespace -sdkPath $sdkPath -cadlInput $cadlFolder/main.cadl -outputJsonFile $newpackageoutput
+        New-CADLPackageFolder `
+            -service $service `
+            -namespace $namespace `
+            -sdkPath $sdkPath `
+            -relatedCadlProjectFolder $cadlRelativeFolder `
+            -specRoot $swaggerDir `
+            -outputJsonFile $newpackageoutput
         $newPackageOutputJson = Get-Content $newPackageOutput -Raw | ConvertFrom-Json
         $relativeSdkPath = $newPackageOutputJson.path
-        npm install
-        npx cadl compile --output-path $sdkPath --emit @azure-tools/cadl-csharp .
-        if ( !$?) {
-            Throw "Failed to generate sdk for cadl. exit code: $?"
-        }
-        GeneratePackage -projectFolder $projectFolder -sdkRootPath $sdkPath -path $relativeSdkPath -downloadUrlPrefix $downloadUrlPrefix -skipGenerate -generatedSDKPackages $generatedSDKPackages
-        Pop-Location
+        GeneratePackage `
+            -projectFolder $projectFolder `
+            -sdkRootPath $sdkPath `
+            -path $relativeSdkPath `
+            -downloadUrlPrefix $downloadUrlPrefix `
+            -generatedSDKPackages $generatedSDKPackages
     }
 }
 $outputJson = [PSCustomObject]@{
diff --git a/eng/scripts/automation/GenerateAndBuildLib.ps1 b/eng/scripts/automation/GenerateAndBuildLib.ps1
@@ -1,5 +1,6 @@
 #Requires -Version 7.0
 $CI_YAML_FILE = "ci.yml"
+$CADL_LOCATION_FILE = "cadl-location.yaml"
 
 . (Join-Path $PSScriptRoot ".." ".." "common" "scripts" "Helpers" PSModule-Helpers.ps1)
 
@@ -359,12 +360,59 @@ function New-MgmtPackageFolder() {
     return $projectFolder
 }
 
+function CreateOrUpdateCadlConfigFile() {
+    param (
+        [string]$cadlConfigurationFile,
+        [string]$directory,
+        [string]$commit = "",
+        [string]$repo = "",
+        [string]$specRoot = ""
+    )
+    if (!(Test-Path -Path $cadlConfigurationFile)) {
+        New-Item -Path $cadlConfigurationFile
+    }
+
+    $configuration = Get-Content -Path $cadlConfigurationFile -Raw | ConvertFrom-Yaml
+    if ( !$configuration) {
+        $configuration = [System.Collections.Generic.Dictionary[string,string]](New-Object 'System.Collections.Generic.Dictionary[string,string]')
+    }
+    $configuration["directory"] = $directory
+    if ($commit) {
+        $configuration["commit"] = $commit
+    } else {
+        $configuration.Remove("commit")
+    }
+    if ($repo) {
+        $configuration["repo"] = $repo
+    } else {
+        $configuration.Remove("repo")
+    }
+
+    if ($specRoot) {
+        $configuration["spec-root-dir"] = $specRoot
+    } else {
+        $configuration.Remove("spec-root-dir")
+    }
+
+    $fileContent = ""
+    foreach ( $key in $configuration.keys) {
+        if ($configuration[$key]) {
+            $fileContent += ( $key + ": " + $configuration[$key] + [Environment]::NewLine)
+        }
+    }
+
+    $fileContent | Out-File $cadlConfigurationFile
+}
+
 function New-CADLPackageFolder() {
     param(
         [string]$service,
         [string]$namespace,
         [string]$sdkPath = "",
-        [string]$cadlInput ="",
+        [string]$relatedCadlProjectFolder,
+        [string]$commit = "",
+        [string]$repo = "",
+        [string]$specRoot = "",
         [string]$outputJsonFile = "$PWD/output.json"
     )
     $serviceFolder = (Join-Path $sdkPath "sdk" $service)
@@ -381,17 +429,14 @@ function New-CADLPackageFolder() {
         if (Test-Path -Path $projectFolder/src/autorest.md) {
             Remove-Item -Path $projectFolder/src/autorest.md
         }
-        $projFile = (Join-Path $projectFolder "src" "$namespace.csproj")
-        $fileContent = Get-Content -Path $projFile
-        $match = ($fileContent | Select-String -Pattern "<AutoRestInput>").LineNumber
-        if ($match.count -gt 0) {
-            $fileContent[$match[0] - 1] = "<AutoRestInput>$cadlInput</AutoRestInput>";
-        } else {
-            $startNum = ($fileContent | Select-String -Pattern '</PropertyGroup>').LineNumber[0]
-            $fileContent[$startNum - 2] += ([Environment]::NewLine + "<AutoRestInput>$cadlInput</AutoRestInput>")
-        }
-        $fileContent | Out-File $projFile
-
+        
+        CreateOrUpdateCadlConfigFile `
+            -cadlConfigurationFile $projectFolder/src/$CADL_LOCATION_FILE `
+            -directory $relatedCadlProjectFolder `
+            -commit $commit `
+            -repo $repo `
+            -specRoot $specRoot
+        
         Update-CIYmlFile -ciFilePath $ciymlFilePath -artifact $namespace
     } else {
         Write-Host "Path doesn't exist. create template."
@@ -425,8 +470,6 @@ function New-CADLPackageFolder() {
         $projFile = (Join-Path $projectFolder "src" "$namespace.csproj")
         $fileContent = Get-Content -Path $projFile
         $fileContent = $fileContent -replace '<Version>[^<]+</Version>', '<Version>1.0.0-beta.1</Version>'
-        $startNum = ($fileContent | Select-String -Pattern '</PropertyGroup>').LineNumber[0]
-        $fileContent[$startNum - 2] += ([Environment]::NewLine + "<AutoRestInput>$cadlInput</AutoRestInput>")
         $fileContent | Out-File $projFile
         # (Get-Content $projFile) -replace "<Version>*.*.*-*.*</Version>", "<Version>1.0.0-beta.1</Version>" | -replace "<AutoRestInput>*</AutoRestInput>", "<AutoRestInput>$cadlInput</AutoRestInput>" |Set-Content $projFile
         Pop-Location
@@ -435,6 +478,14 @@ function New-CADLPackageFolder() {
         if (Test-Path -Path $projectFolder/src/autorest.md) {
             Remove-Item -Path $projectFolder/src/autorest.md
         }
+
+        CreateOrUpdateCadlConfigFile `
+            -cadlConfigurationFile $projectFolder/src/$CADL_LOCATION_FILE `
+            -directory $relatedCadlProjectFolder `
+            -commit $commit `
+            -repo $repo `
+            -specRoot $specRoot
+
         dotnet sln remove src/$namespace.csproj
         dotnet sln add src/$namespace.csproj
         dotnet sln remove tests/$namespace.Tests.csproj
diff --git a/eng/scripts/automation/Invoke-GenerateAndBuild.ps1 b/eng/scripts/automation/Invoke-GenerateAndBuild.ps1
@@ -69,8 +69,6 @@ if ($relatedCadlProjectFolder) {
   $cadlFolder = Resolve-Path (Join-Path $swaggerDir $relatedCadlProjectFolder)
   $newPackageOutput = "newPackageOutput.json"
 
-  Push-Location $cadlFolder
-  trap {Pop-Location}
   $cadlProjectYaml = Get-Content -Path (Join-Path "$cadlFolder" "cadl-project.yaml") -Raw
 
   Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
@@ -85,16 +83,21 @@ if ($relatedCadlProjectFolder) {
       $service = $directories[-3];
       $namespace = $directories[-2];
   }
-  New-CADLPackageFolder -service $service -namespace $namespace -sdkPath $sdkPath -cadlInput $cadlFolder/main.cadl -outputJsonFile $newpackageoutput
+  New-CADLPackageFolder `
+      -service $service `
+      -namespace $namespace `
+      -sdkPath $sdkPath `
+      -relatedCadlProjectFolder $relatedCadlProjectFolder `
+      -specRoot $swaggerDir `
+      -outputJsonFile $newpackageoutput
   $newPackageOutputJson = Get-Content $newPackageOutput -Raw | ConvertFrom-Json
   $relativeSdkPath = $newPackageOutputJson.path
-  npm install
-  npx cadl compile --output-path $sdkPath --emit @azure-tools/cadl-csharp .
-  if ( !$?) {
-      Throw "Failed to generate sdk for cadl. exit code: $?"
-  }
-  GeneratePackage -projectFolder $projectFolder -sdkRootPath $sdkPath -path $relativeSdkPath -downloadUrlPrefix $downloadUrlPrefix -skipGenerate -generatedSDKPackages $generatedSDKPackages
-  Pop-Location
+  GeneratePackage `
+      -projectFolder $projectFolder `
+      -sdkRootPath $sdkPath `
+      -path $relativeSdkPath `
+      -downloadUrlPrefix $downloadUrlPrefix `
+      -generatedSDKPackages $generatedSDKPackages
 }
 $outputJson = [PSCustomObject]@{
   packages = $generatedSDKPackages
