feat(modules)!: enable cloudbuildv2 repository support on tf_cloudbuild_builder and tf_cloudbuild_workspace (#299)

Co-authored-by: Daniel Andrade <dandrade@ciandt.com>

Author: caetano-colin

build/int.cloudbuild.yaml

@@ -89,7 +89,7 @@ steps:
 
 - id: apply-tfsource
   waitFor:
-      - create-all
+      - destroy-simple-folder
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildSourceSimple --stage apply --verbose']
 - id: verify-tfsource
@@ -119,6 +119,44 @@ steps:
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilder --stage teardown --verbose']
 
+- id: apply-tfbuilder-github
+  waitFor:
+      - create-all
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilderGitHub --stage apply --verbose']
+  secretEnv: ['IM_GITHUB_PAT']
+- id: verify-tfbuilder-github
+  waitFor:
+      - apply-tfbuilder-github
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilderGitHub --stage verify --verbose']
+  secretEnv: ['IM_GITHUB_PAT']
+- id: teardown-tfbuilder-github
+  waitFor:
+      - verify-tfbuilder-github
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilderGitHub --stage teardown --verbose']
+  secretEnv: ['IM_GITHUB_PAT']
+
+- id: apply-tfbuilder-gitlab
+  waitFor:
+      - create-all
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilderGitLab --stage apply --verbose']
+  secretEnv: ['IM_GITLAB_PAT']
+- id: verify-tfbuilder-gitlab
+  waitFor:
+      - apply-tfbuilder-gitlab
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilderGitLab --stage verify --verbose']
+  secretEnv: ['IM_GITLAB_PAT']
+- id: teardown-tfbuilder-gitlab
+  waitFor:
+      - verify-tfbuilder-gitlab
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilderGitLab --stage teardown --verbose']
+  secretEnv: ['IM_GITLAB_PAT']
+
 - id: apply-tfworkspace
   waitFor:
       - create-all
@@ -173,6 +211,44 @@ steps:
   args: ['/bin/bash', '-c', 'cft test run TestIMCloudBuildWorkspaceGitLab --stage teardown --verbose']
   secretEnv: ['IM_GITLAB_PAT']
 
+- id: apply-tfworkspace-github
+  waitFor:
+      - create-all
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitHub --stage apply --verbose']
+  secretEnv: ['IM_GITHUB_PAT']
+- id: verify-tfworkspace-github
+  waitFor:
+      - apply-tfworkspace-github
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitHub --stage verify --verbose']
+  secretEnv: ['IM_GITHUB_PAT']
+- id: teardown-tfworkspace-github
+  waitFor:
+      - verify-tfworkspace-github
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitHub --stage teardown --verbose']
+  secretEnv: ['IM_GITHUB_PAT']
+
+- id: apply-tfworkspace-gitlab
+  waitFor:
+      - create-all
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitLab --stage apply --verbose']
+  secretEnv: ['IM_GITLAB_PAT']
+- id: verify-tfworkspace-gitlab
+  waitFor:
+      - apply-tfworkspace-gitlab
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitLab --stage verify --verbose']
+  secretEnv: ['IM_GITLAB_PAT']
+- id: teardown-tfworkspace-gitlab
+  waitFor:
+      - verify-tfworkspace-gitlab
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitLab --stage teardown --verbose']
+  secretEnv: ['IM_GITLAB_PAT']
+
 availableSecrets:
   secretManager:
   - versionName: $_IM_GITHUB_PAT_SECRET_ID/versions/latest

docs/upgrading_to_v9.0.md

@@ -0,0 +1,42 @@
+# Upgrading to v9.0
+
+The v9.0 release of *bootstrap* is a backwards incompatible release.
+
+Some variables default values were removed to align with the restriction that Cloud Build Repositories (2nd Gen) cannot be created in multi-regions or in the `global` region.
+
+You need to update your configurations if you used the default values to prevent resources to be recreated.
+
+## Default value for variable `trigger_location` in module `tf_cloudbuild_workspace` was removed
+
+To preserve the resources created before. include the input `trigger_location` with the previous default value in the module call
+
+```diff
+module "tf_workspace" {
+  source  = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_workspace"
+- version = "~> 8.0"
++ version = "~> 9.0"
+
++ trigger_location = "global"
+```
+
+## Default value for variables `trigger_location` and `gar_repo_location` in module `tf_cloudbuild_builde` were removed
+
+To preserve the resources created before, include the inputs `trigger_location` and `gar_repo_location` with the previous default values in the module call
+
+```diff
+module "cloudbuilder" {
+  source  = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_builder"
+- version = "~> 8.0"
++ version = "~> 9.0"
+
++ trigger_location  = "global"
++ gar_repo_location = "us"
+```
+
+An apply after adding the two inputs will still have an *in-place update* in the `google_workflows_workflow` created by the module.
+
+The endpoint that is used to trigger a build was replaced with a new one that allows a location to be provided.
+
+```
+ # module.cloudbuilder.google_workflows_workflow.builder will be updated in-place
+```

examples/tf_cloudbuild_builder_simple/main.tf

@@ -20,6 +20,8 @@ module "cloudbuilder" {
 
   project_id          = module.enabled_google_apis.project_id
   dockerfile_repo_uri = google_sourcerepo_repository.builder_dockerfile_repo.url
+  trigger_location    = "us-central1"
+  gar_repo_location   = "us-central1"
   # allow logs bucket to be destroyed
   cb_logs_bucket_force_destroy = true
 }

examples/tf_cloudbuild_builder_simple_github/Dockerfile

@@ -0,0 +1,39 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/cloud-builders/gcloud-slim
+
+# Use ARG so that values can be overriden by user/cloudbuild
+ARG TERRAFORM_VERSION=1.1.0
+
+ENV ENV_TERRAFORM_VERSION=$TERRAFORM_VERSION
+
+RUN apt-get update && \
+    /builder/google-cloud-sdk/bin/gcloud -q components install alpha beta terraform-tools && \
+    apt-get -y install curl jq unzip git ca-certificates gnupg && \
+    curl https://releases.hashicorp.com/terraform/${ENV_TERRAFORM_VERSION}/terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip --output terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip && \
+    curl https://releases.hashicorp.com/terraform/${ENV_TERRAFORM_VERSION}/terraform_${ENV_TERRAFORM_VERSION}_SHA256SUMS.sig --output terraform_SHA256SUMS.sig  && \
+    curl https://releases.hashicorp.com/terraform/${ENV_TERRAFORM_VERSION}/terraform_${ENV_TERRAFORM_VERSION}_SHA256SUMS --output terraform_SHA256SUMS && \
+    curl https://keybase.io/hashicorp/pgp_keys.asc --output pgp_keys.asc && \
+    gpg --import pgp_keys.asc && \
+    gpg --verify terraform_SHA256SUMS.sig terraform_SHA256SUMS && \
+    grep terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip terraform_SHA256SUMS | shasum --algorithm 256 --check  && \
+    unzip terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip -d /builder/terraform && \
+    rm -f terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip terraform_SHA256SUMS && \
+    apt-get --purge -y autoremove && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV PATH=/builder/terraform/:$PATH
+ENTRYPOINT ["terraform"]

examples/tf_cloudbuild_builder_simple_github/README.md

@@ -0,0 +1,34 @@
+## Overview
+
+This example demonstrates the simplest usage of the [tf_cloudbuild_builder](../../modules/tf_cloudbuild_builder/) module with a Cloud Build repositories (2nd gen) GitHub repository.
+
+For GitHub connections you will need:
+
+- Install the [Cloud Build App](https://github.com/apps/google-cloud-build) on Github.
+- Create a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) on Github with [scopes](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes) `repo` and `read:user` (or if app is installed in a organization use `read:org`).
+
+For more information on this topic refer to the Cloud Build repositories (2nd gen) documentation for
+[Connect to a GitHub repository](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github?generation=2nd-gen).
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| github\_pat | GitHub personal access token. | `string` | n/a | yes |
+| project\_id | The ID of the project in which to provision resources. | `string` | n/a | yes |
+| repository\_uri | The URI of the GitHub repository where the Terraform configs are stored. | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| artifact\_repo | GAR Repo created to store TF Cloud Builder images |
+| cloudbuild\_trigger\_id | Trigger used for building new TF Builder |
+| location | The location in which the resources were provisioned |
+| project\_id | The ID of the project in which the resources were provisioned |
+| repository\_id | ID of the Cloud Build repositories (2nd gen) repository |
+| scheduler\_id | Scheduler ID for periodically triggering TF Builder build Workflow |
+| workflow\_id | Workflow ID for triggering new TF Builder build |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/tf_cloudbuild_builder_simple_github/apis.tf

@@ -0,0 +1,32 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "enabled_google_apis" {
+  source  = "terraform-google-modules/project-factory/google//modules/project_services"
+  version = "~> 15.0"
+
+  project_id                  = var.project_id
+  disable_services_on_destroy = false
+
+  activate_apis = [
+    "iam.googleapis.com",
+    "compute.googleapis.com",
+    "workflows.googleapis.com",
+    "artifactregistry.googleapis.com",
+    "cloudbuild.googleapis.com",
+    "cloudscheduler.googleapis.com"
+  ]
+}

examples/tf_cloudbuild_builder_simple_github/main.tf

@@ -0,0 +1,118 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  // Found in the URL of your Cloud Build GitHub app configuration settings
+  // https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github?generation=2nd-gen#connecting_a_github_host_programmatically
+  github_app_installation_id = "47590865"
+
+  # GitHub repo url of form "github.com/owner/name"
+  repoURL              = endswith(var.repository_uri, ".git") ? var.repository_uri : "${var.repository_uri}.git"
+  repoURLWithoutSuffix = trimsuffix(local.repoURL, ".git")
+  gh_repo_url_split    = split("/", local.repoURLWithoutSuffix)
+  gh_name              = local.gh_repo_url_split[length(local.gh_repo_url_split) - 1]
+
+  location = "us-central1"
+}
+
+data "google_project" "project" {
+  project_id = var.project_id
+}
+
+// Added to various IDs to prevent potential conflicts for deployments targeting the same repository.
+resource "random_id" "resources_random_id" {
+  byte_length = 4
+}
+
+module "cloudbuilder" {
+  source  = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_builder"
+  version = "~> 8.0"
+
+  project_id                  = module.enabled_google_apis.project_id
+  dockerfile_repo_uri         = google_cloudbuildv2_repository.repository_connection.id
+  dockerfile_repo_type        = "GITHUB"
+  use_cloudbuildv2_repository = true
+  trigger_location            = local.location
+  gar_repo_location           = local.location
+  bucket_name                 = "tf-cloudbuilder-build-logs-${var.project_id}-gh"
+  gar_repo_name               = "tf-runners-gh"
+  workflow_name               = "terraform-runner-workflow-gh"
+  trigger_name                = "tf-cloud-builder-build-gh"
+
+  # allow logs bucket to be destroyed
+  cb_logs_bucket_force_destroy = true
+}
+
+// Create a secret containing the personal access token and grant permissions to the Service Agent.
+resource "google_secret_manager_secret" "github_token_secret" {
+  project   = var.project_id
+  secret_id = "builder-gh-${random_id.resources_random_id.dec}-${local.gh_name}"
+
+  labels = {
+    label = "builder-gh-${random_id.resources_random_id.dec}"
+  }
+
+  replication {
+    auto {}
+  }
+}
+
+// Personal access token from VCS.
+resource "google_secret_manager_secret_version" "github_token_secret_version" {
+  secret      = google_secret_manager_secret.github_token_secret.id
+  secret_data = var.github_pat
+}
+
+resource "google_secret_manager_secret_iam_member" "github_token_iam_member" {
+  project   = var.project_id
+  secret_id = google_secret_manager_secret.github_token_secret.id
+  role      = "roles/secretmanager.secretAccessor"
+  member    = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-cloudbuild.iam.gserviceaccount.com"
+}
+
+// See https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github?generation=2nd-gen
+resource "google_cloudbuildv2_connection" "vcs_connection" {
+  project  = var.project_id
+  name     = "builder-gh-${random_id.resources_random_id.dec}-${var.project_id}"
+  location = local.location
+
+  github_config {
+    app_installation_id = local.github_app_installation_id
+    authorizer_credential {
+      oauth_token_secret_version = google_secret_manager_secret_version.github_token_secret_version.name
+    }
+  }
+}
+
+// Create the repository connection.
+resource "google_cloudbuildv2_repository" "repository_connection" {
+  project  = var.project_id
+  name     = local.gh_name
+  location = local.location
+
+  parent_connection = google_cloudbuildv2_connection.vcs_connection.name
+  remote_uri        = local.repoURL
+}
+
+# Bootstrap GitHub with Dockerfile
+module "bootstrap_github_repo" {
+  source  = "terraform-google-modules/gcloud/google"
+  version = "~> 3.1"
+  upgrade = false
+
+  create_cmd_entrypoint = "${path.module}/scripts/push-to-repo.sh"
+  create_cmd_body       = "${var.github_pat} ${var.repository_uri} ${path.module}/Dockerfile"
+}