Merge pull request #370 from GoogleCloudPlatform/issue369

feat: use cloud deploy env variables

Author: srinandan

cmd/integrationcli/integrationcli.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"fmt"
+	"internal/apiclient"
 	"internal/cmd"
 	"os"
 )
@@ -29,6 +30,7 @@ var (
 
 func main() {
 	rootCmd := cmd.GetRootCmd()
+	apiclient.SetBuildParams(version, commit, date)
 	rootCmd.Version = fmt.Sprintf("%s date: %s [commit: %.7s]", version, date, commit)
 
 	if err := rootCmd.Execute(); err != nil {

internal/apiclient/bundles.go

@@ -225,7 +225,8 @@ func ExtractTgz(gcsURL string) (folder string, err error) {
 	return folder, nil
 }
 
-func GetCloudDeployGCSLocations(pipeline string, release string) (skaffoldConfigUri string, err error) {
+func GetCloudDeployGCSLocations(cloudDeployProjectId string, cloudDeployLocation string,
+	pipeline string, release string) (skaffoldConfigUri string, err error) {
 	type cloudDeployRelease struct {
 		SkaffoldConfigUri string `json:"skaffoldConfigUri"`
 		TargetArtifacts   map[string]struct {
@@ -242,7 +243,7 @@ func GetCloudDeployGCSLocations(pipeline string, release string) (skaffoldConfig
 	r := cloudDeployRelease{}
 
 	cloudDeployURL := fmt.Sprintf("https://clouddeploy.googleapis.com/v1/projects/%s/locations/%s/deliveryPipelines/%s/releases/%s",
-		GetProjectID(), GetRegion(), pipeline, release)
+		cloudDeployProjectId, cloudDeployLocation, pipeline, release)
 	u, _ := url.Parse(cloudDeployURL)
 
 	ClientPrintHttpResponse.Set(false)

internal/apiclient/options.go

@@ -98,6 +98,8 @@ type clientPrintHttpResponse struct {
 
 var ClientPrintHttpResponse = &clientPrintHttpResponse{enable: true}
 
+var cliVersion, cliCommitSha, cliBuildDate string
+
 // NewIntegrationClient sets up options to invoke Integration APIs
 func NewIntegrationClient(o IntegrationClientOptions) {
 	if options == nil {
@@ -495,3 +497,19 @@ func GetMetadataToken() bool {
 func SetMetadataToken(b bool) {
 	options.MetadataToken = b
 }
+
+func SetBuildParams(version string, commit string, date string) {
+	cliVersion = version
+	cliCommitSha = commit
+	cliBuildDate = date
+}
+
+func GetBuildParams() (version string, commit string, date string) {
+	if cliVersion == "" {
+		cliVersion = "unknown"
+	}
+	if cliCommitSha == "" {
+		cliCommitSha = "unknown"
+	}
+	return cliVersion, cliCommitSha, cliBuildDate
+}

internal/cmd/integrations/apply.go

@@ -49,17 +49,6 @@ var ApplyCmd = &cobra.Command{
 		if err = apiclient.SetRegion(utils.GetStringParam(cmdRegion)); err != nil {
 			return err
 		}
-		if folder == "" && (pipeline == "" || release == "" || outputGCSPath == "") {
-			return fmt.Errorf("atleast one of folder or pipeline, release and outputGCSPath must be supplied")
-		}
-		if folder != "" && (pipeline != "" || release != "" || outputGCSPath != "") {
-			return fmt.Errorf("both folder and pipeline, release and outputGCSPath cannot be supplied")
-		}
-		if (pipeline != "" && (release == "" || outputGCSPath == "")) ||
-			(release != "" && (pipeline == "" && outputGCSPath == "")) ||
-			(outputGCSPath != "" && (pipeline == "" && release == "")) {
-			return fmt.Errorf("release, pipeline and outputGCSPath must be set")
-		}
 		cmd.Flags().VisitAll(func(f *pflag.Flag) {
 			clilog.Debug.Printf("%s: %s\n", f.Name, f.Value)
 		})
@@ -71,7 +60,7 @@ var ApplyCmd = &cobra.Command{
 		var skaffoldConfigUri string
 
 		if folder == "" {
-			skaffoldConfigUri, err = apiclient.GetCloudDeployGCSLocations(pipeline, release)
+			skaffoldConfigUri, err = apiclient.GetCloudDeployGCSLocations(cloudDeployProjectId, cloudDeployLocation, pipeline, release)
 			if err != nil {
 				return err
 			}
@@ -90,6 +79,7 @@ var ApplyCmd = &cobra.Command{
 		}
 
 		createSecret, _ := strconv.ParseBool(utils.GetStringParam(cmd.Flag("create-secret")))
+		cloudDeploy, _ := strconv.ParseBool(utils.GetStringParam(cmd.Flag("cloud-deploy")))
 		grantPermission, _ := strconv.ParseBool(utils.GetStringParam(cmd.Flag("grant-permission")))
 		userLabel := utils.GetStringParam(cmd.Flag("user-label"))
 		wait, _ := strconv.ParseBool(utils.GetStringParam(cmd.Flag("wait")))
@@ -110,6 +100,12 @@ var ApplyCmd = &cobra.Command{
 
 		apiclient.DisableCmdPrintHttpResponse()
 
+		if cloudDeploy {
+			if err = storeCloudDeployVariables(); err != nil {
+				return err
+			}
+		}
+
 		if !skipAuthconfigs {
 			if err = processAuthConfigs(authconfigFolder); err != nil {
 				return err
@@ -160,20 +156,17 @@ Apply scaffold configuration, but skip connectors: ` + GetExample(12) + `
 Apply scaffold configuration and run functional tests: ` + GetExample(18),
 }
 
-var serviceAccountName, serviceAccountProject, encryptionKey, pipeline, release, outputGCSPath string
+var serviceAccountName, serviceAccountProject, encryptionKey, pipeline string
+var release, outputGCSPath, cloudDeployProjectId, cloudDeployLocation string
 
 func init() {
 	var userLabel string
-	grantPermission, createSecret, wait, runTests := false, false, false, false
+	grantPermission, createSecret, wait, runTests, cloudDeploy := false, false, false, false, false
 
 	ApplyCmd.Flags().StringVarP(&folder, "folder", "f",
 		"", "Folder containing scaffolding configuration")
-	ApplyCmd.Flags().StringVarP(&pipeline, "pipeline", "",
-		"", "Cloud Deploy Pipeline name")
-	ApplyCmd.Flags().StringVarP(&release, "release", "",
-		"", "Cloud Deploy Release name")
-	ApplyCmd.Flags().StringVarP(&outputGCSPath, "output-gcs-path", "",
-		"", "Upload a file named results.json containing the results")
+	ApplyCmd.Flags().BoolVarP(&cloudDeploy, "cloud-deploy", "",
+		false, "Deploy using Cloud Deploy; default is false")
 	ApplyCmd.Flags().BoolVarP(&grantPermission, "grant-permission", "g",
 		false, "Grant the service account permission to the GCP resource; default is false")
 	ApplyCmd.Flags().StringVarP(&userLabel, "userlabel", "u",
@@ -748,3 +741,17 @@ func processTestCases(testsFolder string, integrationName string, version string
 	}
 	return nil
 }
+
+func storeCloudDeployVariables() (err error) {
+	pipeline = os.Getenv("CLOUD_DEPLOY_DELIVERY_PIPELINE")
+	release = os.Getenv("CLOUD_DEPLOY_RELEASE_NAME")
+	outputGCSPath = os.Getenv("CLOUD_DEPLOY_OUTPUT_GCS_PATH")
+	cloudDeployProjectId = os.Getenv("CLOUD_DEPLOY_PROJECT_ID")
+	cloudDeployLocation = os.Getenv("CLOUD_DEPLOY_LOCATION")
+
+	if pipeline == "" || release == "" || outputGCSPath == "" || cloudDeployProjectId == "" {
+		return fmt.Errorf("CLOUD_DEPLOY_DELIVERY_PIPELINE, CLOUD_DEPLOY_RELEASE_NAME, CLOUD_DEPLOY_OUTPUT_GCS_PATH, " +
+			"CLOUD_DEPLOY_PROJECT_ID, CLOUD_DEPLOY_LOCATION must be set")
+	}
+	return nil
+}

internal/cmd/render/render.go

@@ -30,8 +30,8 @@ var Cmd = &cobra.Command{
 		cmd.SilenceUsage = true
 
 		outputGCSPath := utils.GetStringParam(cmd.Flag("output-gcs-path"))
-
-		err = apiclient.WriteManifest(outputGCSPath, utils.GetCLIVersion())
+		v, _, _ := apiclient.GetBuildParams()
+		err = apiclient.WriteManifest(outputGCSPath, v)
 		return
 	},
 }

internal/cmd/utils/utils.go

@@ -19,7 +19,6 @@ import (
 	"internal/apiclient"
 	"io"
 	"os"
-	"runtime/debug"
 	"strconv"
 	"strings"
 
@@ -51,7 +50,7 @@ const cloudBuild = `# Copyright 2023 Google LLC
 
 steps:
 - id: 'Apply Integration scaffolding configuration'
-  name: us-docker.pkg.dev/appintegration-toolkit/images/integrationcli:latest
+  name: us-docker.pkg.dev/appintegration-toolkit/images/integrationcli:%s
   args:
     - integrations
     - apply
@@ -104,26 +103,27 @@ metadata:
   name: appint-%s-pipeline
 serialPipeline:
   stages:
-  - targetId: %s-env
+  - targetId: %s
 ---
 
 apiVersion: deploy.cloud.google.com/v1
 kind: Target
 metadata:
-  name: %s-env
+  name: %s
 customTarget:
   customTargetType: appint-%s-target
 deployParameters:
   APP_INTEGRATION_PROJECT_ID: "%s"
+  APP_INTEGRATION_REGION: "%s"
 ---
 
 apiVersion: deploy.cloud.google.com/v1
 kind: CustomTargetType
 metadata:
   name: appint-%s-target
 customActions:
-  renderAction: render-app-integration
-  deployAction: deploy-app-integration`
+  renderAction: render-%s-integration
+  deployAction: deploy-%s-integration`
 
 var skaffold = `# Copyright 2024 Google LLC
 #
@@ -145,7 +145,7 @@ customActions:
 - name: render-%s-integration
   containers:
   - name: render
-    image: us-docker.pkg.dev/appintegration-toolkit/images/integrationcli:latest
+    image: us-docker.pkg.dev/appintegration-toolkit/images/integrationcli:%s
     command: ['sh']
     args:
       - '-c'
@@ -154,12 +154,12 @@ customActions:
 - name: deploy-%s-integration
   containers:
   - name: deploy
-    image: us-docker.pkg.dev/appintegration-toolkit/images/integrationcli:latest
+    image: us-docker.pkg.dev/appintegration-toolkit/images/integrationcli:%s
     command: ['sh']
     args:
       - '-c'
       - |-
-        integrationcli integrations apply --env=dev --reg=$CLOUD_DEPLOY_LOCATION --proj=$APP_INTEGRATION_PROJECT_ID --pipeline=$CLOUD_DEPLOY_DELIVERY_PIPELINE --release=$CLOUD_DEPLOY_OUTPUT_GCS_PATH --output-gcs-path=$CLOUD_DEPLOY_TARGET --metadata-token`
+        integrationcli integrations apply --env=$CLOUD_DEPLOY_TARGET --reg=$CLOUD_DEPLOY_LOCATION --proj=$APP_INTEGRATION_PROJECT_ID --reg=$APP_INTEGRATION_REGION --cloud-deploy=true --run-tests=true --wait=true --metadata-token`
 
 var githubActionApply = `# Copyright 2025 Google LLC
 #
@@ -223,23 +223,26 @@ jobs:
 
       - name: Create and Publish Integration
         id: 'publish-integration'
-        uses: docker://us-docker.pkg.dev/appintegration-toolkit/images/integrationcli:v0.79.0 #pin to version of choice
+        uses: docker://us-docker.pkg.dev/appintegration-toolkit/images/integrationcli:%s #pin to version of choice
         with:
           args: integrations apply --env=${{ env.ENVIRONMENT}} --folder=. --userlabel=${{ steps.calc-vars.outputs.SHORT_SHA }} --wait=true --proj=${{ env.PROJECT_ID }} --reg=${{ env.REGION }} --token ${{ steps.gcp-auth.outputs.access_token }}`
 
 func GetCloudDeployYaml(integrationName string, env string) string {
 	if env == "" {
 		env = "dev"
 	}
-	return fmt.Sprintf(cloudDeploy, integrationName, env, env, integrationName, apiclient.GetProjectID(), integrationName)
+	return fmt.Sprintf(cloudDeploy, integrationName, env, env, integrationName, apiclient.GetProjectID(), apiclient.GetRegion(),
+		integrationName, integrationName, integrationName)
 }
 
 func GetSkaffoldYaml(integrationName string) string {
-	return fmt.Sprintf(skaffold, integrationName, integrationName)
+	v, _, _ := apiclient.GetBuildParams()
+	return fmt.Sprintf(skaffold, integrationName, v, integrationName, v)
 }
 
 func GetCloudBuildYaml() string {
-	return cloudBuild
+	v, _, _ := apiclient.GetBuildParams()
+	return fmt.Sprintf(cloudBuild, v)
 }
 
 func ReadFile(filePath string) (byteValue []byte, err error) {
@@ -271,19 +274,8 @@ func GetGithubAction(environment string, integrationName string) string {
 	} else {
 		githubAction = githubActionApply
 	}
-	v := GetCLIVersion()
-	if v != "" {
-		githubAction = strings.ReplaceAll(githubAction, "v0.79.0", v)
-	}
-	return fmt.Sprintf(githubAction, integrationName)
-}
-
-func GetCLIVersion() string {
-	bi, ok := debug.ReadBuildInfo()
-	if ok && bi.Main.Version != "" {
-		return bi.Main.Version
-	}
-	return "latest"
+	v, _, _ := apiclient.GetBuildParams()
+	return fmt.Sprintf(githubAction, integrationName, v)
 }
 
 func GetBasicInfo(cmd *cobra.Command, flag string) bool {

samples/scaffold-sample/clouddeploy.yaml

@@ -18,17 +18,18 @@ metadata:
   name: appint-sample-pipeline
 serialPipeline:
   stages:
-  - targetId: dev-env
-  - targetId: staging-env
+  - targetId: dev
+  - targetId: staging
 ---
 apiVersion: deploy.cloud.google.com/v1
 kind: Target
 metadata:
-  name: dev-env
+  name: dev
 customTarget:
   customTargetType: appint-sample-target
 deployParameters:
   APP_INTEGRATION_PROJECT_ID: "dev-integration-project"
+  APP_INTEGRATION_REGION: "us-central1"
 #executionConfigs:
 #- usages:
 #  - DEPLOY
@@ -44,11 +45,12 @@ deployParameters:
 apiVersion: deploy.cloud.google.com/v1
 kind: Target
 metadata:
-  name: staging-env
+  name: staging
 customTarget:
   customTargetType: appint-sample-target
 deployParameters:
   APP_INTEGRATION_PROJECT_ID: "staging-integration-project"
+  APP_INTEGRATION_REGION: "us-central1"
 #executionConfigs:
 #- usages:
 #  - DEPLOY

samples/scaffold-sample/skaffold.yaml

@@ -32,4 +32,4 @@ customActions:
     args:
       - '-c'
       - |-
-        integrationcli integrations apply --env=dev --reg=$CLOUD_DEPLOY_LOCATION --proj=$APP_INTEGRATION_PROJECT_ID --pipeline=$CLOUD_DEPLOY_DELIVERY_PIPELINE --release=$CLOUD_DEPLOY_RELEASE --output-gcs-path=$CLOUD_DEPLOY_OUTPUT_GCS_PATH --run-tests=true --metadata-token
+        integrationcli integrations apply --env=$CLOUD_DEPLOY_TARGET --reg=$APP_INTEGRATION_REGION --proj=$APP_INTEGRATION_PROJECT_ID --wait=true --run-tests=true --metadata-token