From 72ef7153d2cd8e2d949effa97a488b51aea092f7 Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Fri, 31 Jan 2025 18:13:55 +0000 Subject: [PATCH 01/20] feat: initial commit composer v3 --- modules/create_environment_v3/README.md | 119 +++++++++ modules/create_environment_v3/iam.tf | 26 ++ modules/create_environment_v3/main.tf | 191 ++++++++++++++ modules/create_environment_v3/outputs.tf | 40 +++ modules/create_environment_v3/variables.tf | 273 +++++++++++++++++++++ modules/create_environment_v3/versions.tf | 40 +++ 6 files changed, 689 insertions(+) create mode 100644 modules/create_environment_v3/README.md create mode 100644 modules/create_environment_v3/iam.tf create mode 100644 modules/create_environment_v3/main.tf create mode 100644 modules/create_environment_v3/outputs.tf create mode 100644 modules/create_environment_v3/variables.tf create mode 100644 modules/create_environment_v3/versions.tf diff --git a/modules/create_environment_v3/README.md b/modules/create_environment_v3/README.md new file mode 100644 index 00000000..2ac1644a --- /dev/null +++ b/modules/create_environment_v3/README.md @@ -0,0 +1,119 @@ +# Module Cloud Composer Environment ([v3](https://cloud.google.com/composer/docs/composer-3/composer-overview)) + +This module is used to create a Cloud Composer v3 environment. + +## Compatibility + +This module is meant for use with Terraform 1.3+ and tested using Terraform 1.3+. If you find incompatibilities using Terraform >=1.3, please open an issue. + + +```hcl +module "simple-composer-environment" { + source = "terraform-google-modules/composer/google//modules/create_environment_v3" + version = "~> 7.0" + project_id = var.project_id + composer_env_name = "test-composer-env" + region = "us-central1" + composer_service_account = var.composer_service_account + network = "test-vpc" + subnetwork = "test-subnet" + grant_sa_agent_permission = false + environment_size = "ENVIRONMENT_SIZE_SMALL" + use_private_environment = true + composer_network_attachment_name = "composer-na" + + scheduler = { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + count = 2 + } + + dag_processor = { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + count = 2 + } + + web_server = { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + } + + worker = { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + min_count = 2 + max_count = 3 + } + + triggerer = { + cpu = 1 + memory_gb = 1 + count = 2 + } +} + +``` + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| airflow\_config\_overrides | Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example "core-dags\_are\_paused\_at\_creation". | `map(string)` | `{}` | no | +| cloud\_composer\_connection\_subnetwork | Subnetwork self-link. When specified, the environment will use Private Service Connect instead of VPC peerings to connect to CloudSQL in the Tenant Project. IP address of psc endpoint is allocated from this subnet | `string` | `null` | no | +| cloud\_composer\_network\_ipv4\_cidr\_block | The CIDR block from which IP range in tenant project will be reserved. Required if VPC peering is used to connect to CloudSql instead of PSC | `string` | `null` | no | +| cloud\_data\_lineage\_integration | Whether or not Dataplex data lineage integration is enabled. Cloud Composer environments in versions composer-2.1.2-airflow-..* and newer) | `bool` | `false` | no | +| cloud\_sql\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for Cloud SQL private service access. Required if VPC peering is used to connect to CloudSql instead of PSC | `string` | `null` | no | +| composer\_env\_name | Name of Cloud Composer Environment | `string` | n/a | yes | +| composer\_service\_account | Service Account for running Cloud Composer. | `string` | `null` | no | +| enable\_ip\_masq\_agent | Deploys 'ip-masq-agent' daemon set in the GKE cluster and defines nonMasqueradeCIDRs equals to pod IP range so IP masquerading is used for all destination addresses, except between pods traffic. | `bool` | `false` | no | +| enable\_private\_endpoint | Configure private access to the cluster endpoint. If true, access to the public endpoint of the GKE cluster is denied | `bool` | `false` | no | +| enable\_privately\_used\_public\_ips | When enabled, IPs from public (non-RFC1918) ranges can be used for pod\_ip\_allocation\_range\_name and service\_ip\_allocation\_range\_name. | `bool` | `false` | no | +| env\_variables | Variables of the airflow environment. | `map(string)` | `{}` | no | +| environment\_size | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. Values for environment size are: `ENVIRONMENT_SIZE_SMALL`, `ENVIRONMENT_SIZE_MEDIUM`, and `ENVIRONMENT_SIZE_LARGE`. | `string` | `"ENVIRONMENT_SIZE_MEDIUM"` | no | +| grant\_sa\_agent\_permission | Cloud Composer relies on Workload Identity as Google API authentication mechanism for Airflow. | `bool` | `true` | no | +| image\_version | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-2.10.2-airflow-2.10.2"` | no | +| kms\_key\_name | Customer-managed Encryption Key fully qualified resource name, i.e. projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key. | `string` | `null` | no | +| labels | The resource labels (a map of key/value pairs) to be applied to the Cloud Composer. | `map(string)` | `{}` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `null` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `null` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | 
list(object({
    cidr_block   = string
    display_name = string
  }))
| `[]` | no | +| master\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for the GKE master. Required when `use_private_environment` and `enable_private_endpoint` is `true` | `string` | `null` | no | +| network | The VPC network to host the composer cluster. | `string` | n/a | yes | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| pod\_ip\_allocation\_range\_name | The name of the subnet secondary range, used to allocate IP addresses for the pods. | `string` | `null` | no | +| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes | +| pypi\_packages | Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. "numpy"). | `map(string)` | `{}` | no | +| region | Region where the Cloud Composer Environment is created. | `string` | `"us-central1"` | no | +| resilience\_mode | Cloud Composer 2.1.15 or newer only. The resilience mode states whether high resilience is enabled for the environment or not. Values for resilience mode are `HIGH_RESILIENCE` for high resilience and `STANDARD_RESILIENCE` for standard resilience | `string` | `null` | no | +| scheduled\_snapshots\_config | The recovery configuration settings for the Cloud Composer environment | 
object({
    enabled                    = optional(bool, false)
    snapshot_location          = optional(string)
    snapshot_creation_schedule = optional(string)
    time_zone                  = optional(string)
  })
| `null` | no | +| scheduler | Configuration for resources used by Airflow schedulers. | 
object({
    cpu        = string
    memory_gb  = number
    storage_gb = number
    count      = number
  })
| 
{
  "count": 2,
  "cpu": 2,
  "memory_gb": 7.5,
  "storage_gb": 5
}
| no | +| service\_ip\_allocation\_range\_name | The name of the subnet secondary range, used to allocate IP addresses for the Services. | `string` | `null` | no | +| storage\_bucket | Name of an existing Cloud Storage bucket to be used by the environment | `string` | `null` | no | +| subnetwork | The name of the subnetwork to host the composer cluster. | `string` | n/a | yes | +| subnetwork\_region | The subnetwork region of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| tags | Tags applied to all nodes. Tags are used to identify valid sources or targets for network firewalls. | `set(string)` | `[]` | no | +| task\_logs\_retention\_storage\_mode | The mode of storage for Airflow workers task logs. Values for storage mode are CLOUD\_LOGGING\_ONLY to only store logs in cloud logging and CLOUD\_LOGGING\_AND\_CLOUD\_STORAGE to store logs in cloud logging and cloud storage. Cloud Composer 2.0.23 or newer only | `string` | `null` | no | +| triggerer | Configuration for resources used by Airflow triggerer | 
object({
    cpu       = string
    memory_gb = number
    count     = number
  })
| `null` | no | +| use\_private\_environment | Create a private environment. | `bool` | `false` | no | +| web\_server | Configuration for resources used by Airflow web server. | 
object({
    cpu        = string
    memory_gb  = number
    storage_gb = number
  })
| 
{
  "cpu": 2,
  "memory_gb": 7.5,
  "storage_gb": 5
}
| no | +| web\_server\_network\_access\_control | The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions are applied | 
list(object({
    allowed_ip_range = string
    description      = string
  }))
| `null` | no | +| worker | Configuration for resources used by Airflow workers. | 
object({
    cpu        = string
    memory_gb  = number
    storage_gb = number
    min_count  = number
    max_count  = number
  })
| 
{
  "cpu": 2,
  "max_count": 6,
  "memory_gb": 7.5,
  "min_count": 2,
  "storage_gb": 5
}
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| airflow\_uri | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. | +| composer\_env | Cloud Composer Environment | +| composer\_env\_id | ID of Cloud Composer Environment. | +| composer\_env\_name | Name of the Cloud Composer Environment. | +| gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. | +| gke\_cluster | Google Kubernetes Engine cluster used to run the Cloud Composer Environment. | + + diff --git a/modules/create_environment_v3/iam.tf b/modules/create_environment_v3/iam.tf new file mode 100644 index 00000000..e09dc52f --- /dev/null +++ b/modules/create_environment_v3/iam.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_project" "project" { + project_id = var.project_id +} + +resource "google_project_iam_member" "composer_agent_service_account" { + count = var.grant_sa_agent_permission ? 1 : 0 + project = data.google_project.project.project_id + role = "roles/composer.ServiceAgentV2Ext" + member = format("serviceAccount:%s", local.cloud_composer_sa) +} diff --git a/modules/create_environment_v3/main.tf b/modules/create_environment_v3/main.tf new file mode 100644 index 00000000..e2700a01 --- /dev/null +++ b/modules/create_environment_v3/main.tf @@ -0,0 +1,191 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id + subnetwork_region = var.subnetwork_region != "" ? var.subnetwork_region : var.region + cloud_composer_sa = format("service-%s@cloudcomposer-accounts.iam.gserviceaccount.com", data.google_project.project.number) +} + +resource "google_composer_environment" "composer_env" { + provider = google-beta + + project = var.project_id + name = var.composer_env_name + region = var.region + labels = var.labels + + dynamic "storage_config" { + for_each = var.storage_bucket != null ? ["storage_config"] : [] + content { + bucket = var.storage_bucket + } + } + + config { + + enable_private_environment = var.use_private_environment # reusing the existing variable name from previous versions + + environment_size = var.environment_size + resilience_mode = var.resilience_mode + + node_config { + service_account = var.composer_service_account + tags = var.tags + network = var.create_network_attachment ? "projects/${local.network_project_id}/global/networks/${var.network}" : null + subnetwork = var.create_network_attachment ? "projects/${local.network_project_id}/regions/${local.subnetwork_region}/subnetworks/${var.subnetwork}" : null + composer_network_attachment = var.create_network_attachment ? null : "projects/{var.project_id}/regions/${var.region}/networkAttachments/${var.composer_network_attachment_name}" + } + + dynamic "software_config" { + for_each = [ + { + airflow_config_overrides = var.airflow_config_overrides + pypi_packages = var.pypi_packages + env_variables = var.env_variables + image_version = var.image_version + web_server_plugins_mode = var.web_server_plugins_mode + }] + content { + airflow_config_overrides = software_config.value["airflow_config_overrides"] + pypi_packages = software_config.value["pypi_packages"] + env_variables = software_config.value["env_variables"] + image_version = software_config.value["image_version"] + web_server_plugins_mode = software_config.value["web_server_plugins_mode"] + dynamic "cloud_data_lineage_integration" { + for_each = var.cloud_data_lineage_integration ? ["cloud_data_lineage_integration"] : [] + content { + enabled = var.cloud_data_lineage_integration + } + } + } + } + + dynamic "maintenance_window" { + for_each = (var.maintenance_end_time != null && var.maintenance_recurrence != null) ? [ + { + start_time = var.maintenance_start_time + end_time = var.maintenance_end_time + recurrence = var.maintenance_recurrence + }] : [] + content { + start_time = maintenance_window.value["start_time"] + end_time = maintenance_window.value["end_time"] + recurrence = maintenance_window.value["recurrence"] + } + } + + workloads_config { + + dynamic "scheduler" { + for_each = var.scheduler != null ? [var.scheduler] : [] + content { + cpu = scheduler.value["cpu"] + memory_gb = scheduler.value["memory_gb"] + storage_gb = scheduler.value["storage_gb"] + count = scheduler.value["count"] + } + } + + dynamic "web_server" { + for_each = var.web_server != null ? [var.web_server] : [] + content { + cpu = web_server.value["cpu"] + memory_gb = web_server.value["memory_gb"] + storage_gb = web_server.value["storage_gb"] + } + } + + dynamic "worker" { + for_each = var.worker != null ? [var.worker] : [] + content { + cpu = worker.value["cpu"] + memory_gb = worker.value["memory_gb"] + storage_gb = worker.value["storage_gb"] + min_count = worker.value["min_count"] + max_count = worker.value["max_count"] + } + } + + dynamic "triggerer" { + for_each = var.triggerer != null ? [var.triggerer] : [] + content { + cpu = triggerer.value["cpu"] + memory_gb = triggerer.value["memory_gb"] + count = triggerer.value["count"] + } + } + + dynamic "dag_processor" { + for_each = var.dag_processor != null ? [var.dag_processor] : [] + content { + cpu = dag_processor.value["cpu"] + memory_gb = dag_processor.value["memory_gb"] + storage_gb = dag_processor.value["storage_gb"] + count = dag_processor.value["count"] + } + } + + } + + dynamic "recovery_config" { + for_each = var.scheduled_snapshots_config != null ? ["recovery_config"] : [] + content { + dynamic "scheduled_snapshots_config" { + for_each = var.scheduled_snapshots_config != null ? [var.scheduled_snapshots_config] : [] + content { + enabled = scheduled_snapshots_config.value["enabled"] + snapshot_location = scheduled_snapshots_config.value["snapshot_location"] + snapshot_creation_schedule = scheduled_snapshots_config.value["snapshot_creation_schedule"] + time_zone = scheduled_snapshots_config.value["time_zone"] + } + } + } + } + + dynamic "web_server_network_access_control" { + for_each = var.web_server_network_access_control == null ? [] : ["web_server_network_access_control"] + content { + dynamic "allowed_ip_range" { + for_each = { for x in var.web_server_network_access_control : x.allowed_ip_range => x } + content { + value = allowed_ip_range.value["allowed_ip_range"] + description = allowed_ip_range.value["description"] + } + } + } + } + + dynamic "encryption_config" { + for_each = var.kms_key_name != null ? ["encryption_config"] : [] + content { + kms_key_name = var.kms_key_name + } + } + + dynamic "data_retention_config" { + for_each = var.task_logs_retention_storage_mode == null ? [] : ["data_retention_config"] + content { + task_logs_retention_config { + storage_mode = var.task_logs_retention_storage_mode + } + } + } + } + + depends_on = [google_project_iam_member.composer_agent_service_account] + +} diff --git a/modules/create_environment_v3/outputs.tf b/modules/create_environment_v3/outputs.tf new file mode 100644 index 00000000..41d33bc0 --- /dev/null +++ b/modules/create_environment_v3/outputs.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "composer_env_name" { + value = google_composer_environment.composer_env.name + description = "Name of the Cloud Composer Environment." +} + +output "composer_env_id" { + value = google_composer_environment.composer_env.id + description = "ID of Cloud Composer Environment." +} + +output "gcs_bucket" { + value = google_composer_environment.composer_env.config[0].dag_gcs_prefix + description = "Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment." +} + +output "airflow_uri" { + value = google_composer_environment.composer_env.config[0].airflow_uri + description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment." +} + +output "composer_env" { + value = google_composer_environment.composer_env + description = "Cloud Composer Environment" +} diff --git a/modules/create_environment_v3/variables.tf b/modules/create_environment_v3/variables.tf new file mode 100644 index 00000000..26652b2c --- /dev/null +++ b/modules/create_environment_v3/variables.tf @@ -0,0 +1,273 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project ID where Cloud Composer Environment is created." + type = string +} + +variable "composer_env_name" { + description = "Name of Cloud Composer Environment" + type = string +} + +variable "region" { + description = "Region where the Cloud Composer Environment is created." + type = string + default = "us-central1" +} + +variable "labels" { + type = map(string) + description = "The resource labels (a map of key/value pairs) to be applied to the Cloud Composer." + default = {} +} + +variable "tags" { + description = "Tags applied to all nodes. Tags are used to identify valid sources or targets for network firewalls." + type = set(string) + default = [] +} + +variable "network" { + type = string + description = "The VPC network to host the composer cluster." +} + +variable "network_project_id" { + type = string + description = "The project ID of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "subnetwork" { + type = string + description = "The name of the subnetwork to host the composer cluster." +} + +variable "subnetwork_region" { + type = string + description = "The subnetwork region of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "create_network_attachment" { + type = bool + description = "Either create a new network attachment or use existing one. If true, provide the subnet details." + default = true +} + +variable "composer_network_attachment_name" { + type = string + description = "Name for PSC (Private Service Connect) Network entry point." + default = null +} + +variable "composer_service_account" { + description = "Service Account for running Cloud Composer." + type = string + default = null +} + +variable "airflow_config_overrides" { + type = map(string) + description = "Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example \"core-dags_are_paused_at_creation\"." + default = {} +} + +variable "env_variables" { + type = map(string) + description = "Variables of the airflow environment." + default = {} +} + +variable "image_version" { + type = string + description = "The version of the aiflow running in the cloud composer environment." + default = "composer-3-airflow-2.10.2-build.5" +} + +variable "web_server_plugins_mode" { + type = string + description = "Web server plugins configuration. Can be either 'ENABLED' or 'DISABLED'. Defaults to 'ENABLED'." + default = "ENABLED" +} + +variable "pypi_packages" { + type = map(string) + description = " Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. \"numpy\")." + default = {} +} + +variable "use_private_environment" { + description = "Create a private environment. If true, a private Composer environment will be created." + type = bool + default = false +} + +variable "maintenance_start_time" { + description = "Time window specified for daily or recurring maintenance operations in RFC3339 format" + type = string + default = "05:00" +} + +variable "maintenance_end_time" { + description = "Time window specified for recurring maintenance operations in RFC3339 format" + type = string + default = null +} + +variable "maintenance_recurrence" { + description = "Frequency of the recurring maintenance window in RFC5545 format." + type = string + default = null +} + +variable "environment_size" { + type = string + description = "The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. Values for environment size are: `ENVIRONMENT_SIZE_SMALL`, `ENVIRONMENT_SIZE_MEDIUM`, and `ENVIRONMENT_SIZE_LARGE`." + default = "ENVIRONMENT_SIZE_MEDIUM" +} + +variable "scheduler" { + type = object({ + cpu = string + memory_gb = number + storage_gb = number + count = number + }) + default = { + cpu = 1 + memory_gb = 4 + storage_gb = 5 + count = 2 + } + description = "Configuration for resources used by Airflow schedulers." +} + +variable "web_server" { + type = object({ + cpu = string + memory_gb = number + storage_gb = number + }) + default = { + cpu = 2 + memory_gb = 7.5 + storage_gb = 5 + } + description = "Configuration for resources used by Airflow web server." +} + +variable "worker" { + type = object({ + cpu = string + memory_gb = number + storage_gb = number + min_count = number + max_count = number + }) + default = { + cpu = 2 + memory_gb = 7.5 + storage_gb = 5 + min_count = 2 + max_count = 6 + } + description = "Configuration for resources used by Airflow workers." +} + +variable "triggerer" { + type = object({ + cpu = string + memory_gb = number + count = number + }) + default = null + description = " Configuration for resources used by Airflow triggerer" +} + +variable "dag_processor" { + type = object({ + cpu = string + memory_gb = number + storage_gb = number + count = number + }) + default = { + cpu = 2 + memory_gb = 7.5 + storage_gb = 5 + count = 2 + } + description = "Configuration for resources used by Airflow workers." +} + +variable "grant_sa_agent_permission" { + type = bool + default = true + description = "Cloud Composer relies on Workload Identity as Google API authentication mechanism for Airflow. " +} + +variable "scheduled_snapshots_config" { + type = object({ + enabled = optional(bool, false) + snapshot_location = optional(string) + snapshot_creation_schedule = optional(string) + time_zone = optional(string) + }) + default = null + description = "The recovery configuration settings for the Cloud Composer environment" +} + +variable "storage_bucket" { + description = "Name of an existing Cloud Storage bucket to be used by the environment" + type = string + default = null +} + +variable "resilience_mode" { + description = "Cloud Composer 2.1.15 or newer only. The resilience mode states whether high resilience is enabled for the environment or not. Values for resilience mode are `HIGH_RESILIENCE` for high resilience and `STANDARD_RESILIENCE` for standard resilience" + type = string + default = null +} + +variable "cloud_data_lineage_integration" { + description = "Whether or not Dataplex data lineage integration is enabled. Cloud Composer environments in versions composer-2.1.2-airflow-..* and newer)" + type = bool + default = false +} + +variable "web_server_network_access_control" { + type = list(object({ + allowed_ip_range = string + description = string + })) + default = null + description = "The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions are applied" +} + +variable "kms_key_name" { + description = "Customer-managed Encryption Key fully qualified resource name, i.e. projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key." + type = string + default = null +} + +variable "task_logs_retention_storage_mode" { + description = "The mode of storage for Airflow workers task logs. Values for storage mode are CLOUD_LOGGING_ONLY to only store logs in cloud logging and CLOUD_LOGGING_AND_CLOUD_STORAGE to store logs in cloud logging and cloud storage. Cloud Composer 2.0.23 or newer only" + type = string + default = null +} diff --git a/modules/create_environment_v3/versions.tf b/modules/create_environment_v3/versions.tf new file mode 100644 index 00000000..0685edea --- /dev/null +++ b/modules/create_environment_v3/versions.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.3" + required_providers { + + google = { + source = "hashicorp/google" + version = ">= 6.0, < 7" + } + + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.0, < 7" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/terraform-google-composer:create_environment_v3/v6.0.0" + } + + provider_meta "google-beta" { + module_name = "blueprints/terraform/terraform-google-composer:create_environment_v3/v6.0.0" + } + +} From 4e4dc9ddd5c90253ede1ed25909a5a42d7ff086a Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Fri, 31 Jan 2025 19:08:35 +0000 Subject: [PATCH 02/20] feat: added examples and tests --- examples/simple_composer_env_v3/README.md | 34 +++++ examples/simple_composer_env_v3/main.tf | 116 ++++++++++++++++++ examples/simple_composer_env_v3/outputs.tf | 35 ++++++ examples/simple_composer_env_v3/variables.tf | 46 +++++++ modules/create_environment_v3/README.md | 21 ++-- .../simple_composer_env_v3_test.go | 40 ++++++ 6 files changed, 278 insertions(+), 14 deletions(-) create mode 100644 examples/simple_composer_env_v3/README.md create mode 100644 examples/simple_composer_env_v3/main.tf create mode 100644 examples/simple_composer_env_v3/outputs.tf create mode 100644 examples/simple_composer_env_v3/variables.tf create mode 100644 test/integration/simple-composer-env-v3/simple_composer_env_v3_test.go diff --git a/examples/simple_composer_env_v3/README.md b/examples/simple_composer_env_v3/README.md new file mode 100644 index 00000000..c7465e70 --- /dev/null +++ b/examples/simple_composer_env_v3/README.md @@ -0,0 +1,34 @@ +# Simple Cloud Composer Environment (V3) Example + +This example illustrates how to use the `composer` V2 module to deploy private composer environment with private service connect (PSC) endpoint to connect network attachments. + +This example also creates a Cloud Storage Bucket for scheduled snapshots and assign appropriate permission(s) to Composer Service Account on the bucket. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| composer\_env\_name | Name of Cloud Composer Environment. | `string` | `"ci-composer"` | no | +| composer\_service\_account | Service Account to be used for running Cloud Composer Environment. | `string` | n/a | yes | +| network | Network where Cloud Composer is created. | `string` | n/a | yes | +| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes | +| region | Region where Cloud Composer Environment is created. | `string` | n/a | yes | +| subnetwork | Name of the Subetwork where Cloud Composer is created. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| airflow\_uri | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. | +| composer\_env\_id | ID of Cloud Composer Environment. | +| composer\_env\_name | Name of the Cloud Composer Environment. | +| gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_composer_env_v3/main.tf b/examples/simple_composer_env_v3/main.tf new file mode 100644 index 00000000..a3f2678a --- /dev/null +++ b/examples/simple_composer_env_v3/main.tf @@ -0,0 +1,116 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "key_suffix" { + length = 5 + special = false + upper = false +} + +# Create a bucket to store the snapshots +resource "google_storage_bucket" "my_bucket" { + project = var.project_id + name = "snapshot-bucket-${random_string.key_suffix.result}" + location = var.region + force_destroy = true + uniform_bucket_level_access = true +} + +resource "google_storage_bucket_iam_member" "object_admin" { + bucket = google_storage_bucket.my_bucket.name + role = "roles/storage.objectAdmin" + member = "serviceAccount:${var.composer_service_account}" +} + +module "simple-composer-environment" { + source = "terraform-google-modules/composer/google//modules/create_environment_v3" + version = "~> 6.0" + + project_id = var.project_id + composer_env_name = var.composer_env_name + region = var.region + composer_service_account = var.composer_service_account + network = var.network + subnetwork = var.subnetwork + create_network_attachment = true + + grant_sa_agent_permission = false + environment_size = "ENVIRONMENT_SIZE_SMALL" + + use_private_environment = true + cloud_data_lineage_integration = true + resilience_mode = "STANDARD_RESILIENCE" + + scheduler = { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + count = 2 + } + + dag_processor = { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + count = 2 + } + + web_server = { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + } + + worker = { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + min_count = 2 + max_count = 3 + } + + triggerer = { + cpu = 1 + memory_gb = 1 + count = 2 + } + + scheduled_snapshots_config = { + enabled = true + snapshot_location = google_storage_bucket.my_bucket.url + snapshot_creation_schedule = "0 4 * * *" + time_zone = "UTC+01" + } + + maintenance_start_time = "2025-02-01T00:00:00Z" + maintenance_end_time = "2025-03-01T12:00:00Z" + maintenance_recurrence = "FREQ=WEEKLY;BYDAY=SU,SA" + + depends_on = [ + google_storage_bucket_iam_member.object_admin, + ] + + web_server_network_access_control = [ + { + allowed_ip_range = "192.0.2.0/24" + description = "office net 1" + }, + { + allowed_ip_range = "192.0.4.0/24" + description = "office net 2" + }, + ] +} diff --git a/examples/simple_composer_env_v3/outputs.tf b/examples/simple_composer_env_v3/outputs.tf new file mode 100644 index 00000000..7a24325b --- /dev/null +++ b/examples/simple_composer_env_v3/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "composer_env_name" { + description = "Name of the Cloud Composer Environment." + value = module.simple-composer-environment.composer_env_name +} + +output "composer_env_id" { + description = "ID of Cloud Composer Environment." + value = module.simple-composer-environment.composer_env_id +} + +output "gcs_bucket" { + description = "Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment." + value = module.simple-composer-environment.gcs_bucket +} + +output "airflow_uri" { + description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment." + value = module.simple-composer-environment.airflow_uri +} diff --git a/examples/simple_composer_env_v3/variables.tf b/examples/simple_composer_env_v3/variables.tf new file mode 100644 index 00000000..4ef8fd66 --- /dev/null +++ b/examples/simple_composer_env_v3/variables.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project ID where Cloud Composer Environment is created." + type = string +} + +variable "composer_env_name" { + description = "Name of Cloud Composer Environment." + default = "ci-composer" + type = string +} + +variable "region" { + description = "Region where Cloud Composer Environment is created." + type = string +} + +variable "composer_service_account" { + description = "Service Account to be used for running Cloud Composer Environment." + type = string +} + +variable "network" { + description = "Network where Cloud Composer is created." + type = string +} + +variable "subnetwork" { + description = "Name of the Subetwork where Cloud Composer is created." + type = string +} diff --git a/modules/create_environment_v3/README.md b/modules/create_environment_v3/README.md index 2ac1644a..81b8759b 100644 --- a/modules/create_environment_v3/README.md +++ b/modules/create_environment_v3/README.md @@ -64,45 +64,39 @@ module "simple-composer-environment" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | airflow\_config\_overrides | Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example "core-dags\_are\_paused\_at\_creation". | `map(string)` | `{}` | no | -| cloud\_composer\_connection\_subnetwork | Subnetwork self-link. When specified, the environment will use Private Service Connect instead of VPC peerings to connect to CloudSQL in the Tenant Project. IP address of psc endpoint is allocated from this subnet | `string` | `null` | no | -| cloud\_composer\_network\_ipv4\_cidr\_block | The CIDR block from which IP range in tenant project will be reserved. Required if VPC peering is used to connect to CloudSql instead of PSC | `string` | `null` | no | | cloud\_data\_lineage\_integration | Whether or not Dataplex data lineage integration is enabled. Cloud Composer environments in versions composer-2.1.2-airflow-..* and newer) | `bool` | `false` | no | -| cloud\_sql\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for Cloud SQL private service access. Required if VPC peering is used to connect to CloudSql instead of PSC | `string` | `null` | no | | composer\_env\_name | Name of Cloud Composer Environment | `string` | n/a | yes | +| composer\_network\_attachment\_name | Name for PSC (Private Service Connect) Network entry point. | `string` | `null` | no | | composer\_service\_account | Service Account for running Cloud Composer. | `string` | `null` | no | -| enable\_ip\_masq\_agent | Deploys 'ip-masq-agent' daemon set in the GKE cluster and defines nonMasqueradeCIDRs equals to pod IP range so IP masquerading is used for all destination addresses, except between pods traffic. | `bool` | `false` | no | -| enable\_private\_endpoint | Configure private access to the cluster endpoint. If true, access to the public endpoint of the GKE cluster is denied | `bool` | `false` | no | -| enable\_privately\_used\_public\_ips | When enabled, IPs from public (non-RFC1918) ranges can be used for pod\_ip\_allocation\_range\_name and service\_ip\_allocation\_range\_name. | `bool` | `false` | no | +| create\_network\_attachment | Either create a new network attachment or use existing one. If true, provide the subnet details. | `bool` | `true` | no | +| dag\_processor | Configuration for resources used by Airflow workers. | 
object({
    cpu        = string
    memory_gb  = number
    storage_gb = number
    count      = number
  })
| 
{
  "count": 2,
  "cpu": 2,
  "memory_gb": 7.5,
  "storage_gb": 5
}
| no | | env\_variables | Variables of the airflow environment. | `map(string)` | `{}` | no | | environment\_size | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. Values for environment size are: `ENVIRONMENT_SIZE_SMALL`, `ENVIRONMENT_SIZE_MEDIUM`, and `ENVIRONMENT_SIZE_LARGE`. | `string` | `"ENVIRONMENT_SIZE_MEDIUM"` | no | | grant\_sa\_agent\_permission | Cloud Composer relies on Workload Identity as Google API authentication mechanism for Airflow. | `bool` | `true` | no | -| image\_version | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-2.10.2-airflow-2.10.2"` | no | +| image\_version | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-3-airflow-2.10.2-build.5"` | no | | kms\_key\_name | Customer-managed Encryption Key fully qualified resource name, i.e. projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key. | `string` | `null` | no | | labels | The resource labels (a map of key/value pairs) to be applied to the Cloud Composer. | `map(string)` | `{}` | no | | maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `null` | no | | maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `null` | no | | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | 
list(object({
    cidr_block   = string
    display_name = string
  }))
| `[]` | no | -| master\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for the GKE master. Required when `use_private_environment` and `enable_private_endpoint` is `true` | `string` | `null` | no | | network | The VPC network to host the composer cluster. | `string` | n/a | yes | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| pod\_ip\_allocation\_range\_name | The name of the subnet secondary range, used to allocate IP addresses for the pods. | `string` | `null` | no | | project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes | | pypi\_packages | Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. "numpy"). | `map(string)` | `{}` | no | | region | Region where the Cloud Composer Environment is created. | `string` | `"us-central1"` | no | | resilience\_mode | Cloud Composer 2.1.15 or newer only. The resilience mode states whether high resilience is enabled for the environment or not. Values for resilience mode are `HIGH_RESILIENCE` for high resilience and `STANDARD_RESILIENCE` for standard resilience | `string` | `null` | no | | scheduled\_snapshots\_config | The recovery configuration settings for the Cloud Composer environment | 
object({
    enabled                    = optional(bool, false)
    snapshot_location          = optional(string)
    snapshot_creation_schedule = optional(string)
    time_zone                  = optional(string)
  })
| `null` | no | -| scheduler | Configuration for resources used by Airflow schedulers. | 
object({
    cpu        = string
    memory_gb  = number
    storage_gb = number
    count      = number
  })
| 
{
  "count": 2,
  "cpu": 2,
  "memory_gb": 7.5,
  "storage_gb": 5
}
| no | -| service\_ip\_allocation\_range\_name | The name of the subnet secondary range, used to allocate IP addresses for the Services. | `string` | `null` | no | +| scheduler | Configuration for resources used by Airflow schedulers. | 
object({
    cpu        = string
    memory_gb  = number
    storage_gb = number
    count      = number
  })
| 
{
  "count": 2,
  "cpu": 1,
  "memory_gb": 4,
  "storage_gb": 5
}
| no | | storage\_bucket | Name of an existing Cloud Storage bucket to be used by the environment | `string` | `null` | no | | subnetwork | The name of the subnetwork to host the composer cluster. | `string` | n/a | yes | | subnetwork\_region | The subnetwork region of the shared VPC's host (for shared vpc support) | `string` | `""` | no | | tags | Tags applied to all nodes. Tags are used to identify valid sources or targets for network firewalls. | `set(string)` | `[]` | no | | task\_logs\_retention\_storage\_mode | The mode of storage for Airflow workers task logs. Values for storage mode are CLOUD\_LOGGING\_ONLY to only store logs in cloud logging and CLOUD\_LOGGING\_AND\_CLOUD\_STORAGE to store logs in cloud logging and cloud storage. Cloud Composer 2.0.23 or newer only | `string` | `null` | no | | triggerer | Configuration for resources used by Airflow triggerer | 
object({
    cpu       = string
    memory_gb = number
    count     = number
  })
| `null` | no | -| use\_private\_environment | Create a private environment. | `bool` | `false` | no | +| use\_private\_environment | Create a private environment. If true, a private Composer environment will be created. | `bool` | `false` | no | | web\_server | Configuration for resources used by Airflow web server. | 
object({
    cpu        = string
    memory_gb  = number
    storage_gb = number
  })
| 
{
  "cpu": 2,
  "memory_gb": 7.5,
  "storage_gb": 5
}
| no | | web\_server\_network\_access\_control | The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions are applied | 
list(object({
    allowed_ip_range = string
    description      = string
  }))
| `null` | no | +| web\_server\_plugins\_mode | Web server plugins configuration. Can be either 'ENABLED' or 'DISABLED'. Defaults to 'ENABLED'. | `string` | `"ENABLED"` | no | | worker | Configuration for resources used by Airflow workers. | 
object({
    cpu        = string
    memory_gb  = number
    storage_gb = number
    min_count  = number
    max_count  = number
  })
| 
{
  "cpu": 2,
  "max_count": 6,
  "memory_gb": 7.5,
  "min_count": 2,
  "storage_gb": 5
}
| no | ## Outputs @@ -114,6 +108,5 @@ module "simple-composer-environment" { | composer\_env\_id | ID of Cloud Composer Environment. | | composer\_env\_name | Name of the Cloud Composer Environment. | | gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. | -| gke\_cluster | Google Kubernetes Engine cluster used to run the Cloud Composer Environment. | diff --git a/test/integration/simple-composer-env-v3/simple_composer_env_v3_test.go b/test/integration/simple-composer-env-v3/simple_composer_env_v3_test.go new file mode 100644 index 00000000..ed84fe1d --- /dev/null +++ b/test/integration/simple-composer-env-v3/simple_composer_env_v3_test.go @@ -0,0 +1,40 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package simple_composer_env_v3 + +import ( + "fmt" + "testing" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" +) + +func TestSimpleComposerEnvV3Module(t *testing.T) { + composer := tft.NewTFBlueprintTest(t) + + composer.DefineVerify(func(assert *assert.Assertions) { + composer.DefaultVerify(assert) + + projectID := composer.GetStringOutput("project_id") + + op := gcloud.Runf(t, "composer environments describe %s --project=%s --location=us-central1", composer.GetStringOutput("composer_env_name"), projectID) + assert.Equal(fmt.Sprintf("projects/%s/locations/us-central1/environments/%s", projectID, composer.GetStringOutput("composer_env_name")), op.Get("name").String(), "Composer name is valid") + assert.Equal(composer.GetStringOutput("airflow_uri"), op.Get("config.airflowUri").String(), "AirflowUri is valid") + assert.Equal(composer.GetStringOutput("gcs_bucket"), op.Get("config.dagGcsPrefix").String(), "GCS-Dag is valid") + }) + composer.Test() +} From 682cb64747d20bae1927a681b2e5d0b4315bfde2 Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Fri, 31 Jan 2025 19:17:25 +0000 Subject: [PATCH 03/20] chore:updated build with v3 --- build/int.cloudbuild.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 07b56b2a..4b1eb84d 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -78,6 +78,31 @@ steps: name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage destroy --verbose'] + + # ----- SUITE simple-composer-env-v3 + +- id: init-simple-composer-env-v3 + waitFor: + - create all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage init --verbose'] +- id: apply-simple-composer-env-v3 + waitFor: + - init-simple-composer-env-v3 + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage apply --verbose'] +- id: verify-simple-composer-env-v3 + waitFor: + - apply-simple-composer-env-v3 + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage verify --verbose'] +- id: destroy-simple-composer-env-v3 + waitFor: + - verify-simple-composer-env-v3 + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage destroy --verbose'] + + tags: - 'ci' - 'integration' From dde0c40032c34469ae3c11f219c5c2db38d46846 Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Sat, 1 Feb 2025 04:50:49 +0000 Subject: [PATCH 04/20] chore: added test fixtures --- test/fixtures/simple-composer-env-v3/main.tf | 31 ++++++++++++ .../simple-composer-env-v3/network.tf | 46 +++++++++++++++++ .../simple-composer-env-v3/outputs.tf | 50 +++++++++++++++++++ .../simple-composer-env-v3/variables.tf | 31 ++++++++++++ 4 files changed, 158 insertions(+) create mode 100644 test/fixtures/simple-composer-env-v3/main.tf create mode 100644 test/fixtures/simple-composer-env-v3/network.tf create mode 100644 test/fixtures/simple-composer-env-v3/outputs.tf create mode 100644 test/fixtures/simple-composer-env-v3/variables.tf diff --git a/test/fixtures/simple-composer-env-v3/main.tf b/test/fixtures/simple-composer-env-v3/main.tf new file mode 100644 index 00000000..6edc98bc --- /dev/null +++ b/test/fixtures/simple-composer-env-v3/main.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "simple-composer" { + source = "../../../examples/simple_composer_env_v3" + + project_id = var.project_id + composer_env_name = "composer-env-${random_id.random_suffix.hex}" + region = var.region + composer_service_account = var.composer_sa + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + create_network_attachment = true +} + +resource "random_id" "random_suffix" { + byte_length = 2 +} diff --git a/test/fixtures/simple-composer-env-v3/network.tf b/test/fixtures/simple-composer-env-v3/network.tf new file mode 100644 index 00000000..7a236a64 --- /dev/null +++ b/test/fixtures/simple-composer-env-v3/network.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_network" "main" { + project = var.project_id + name = "ci-composer-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + project = var.project_id + name = "ci-composer-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + private_ip_google_access = true + + secondary_ip_range { + range_name = "ci-composer-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "ci-composer-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} diff --git a/test/fixtures/simple-composer-env-v3/outputs.tf b/test/fixtures/simple-composer-env-v3/outputs.tf new file mode 100644 index 00000000..fde1fa10 --- /dev/null +++ b/test/fixtures/simple-composer-env-v3/outputs.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + description = "Project ID where Cloud Composer Environment is created." + value = var.project_id +} + +output "composer_env_name" { + description = "Name of the Cloud Composer Environment." + value = module.simple-composer.composer_env_name +} + +output "network" { + description = "The Cloud Composer Network." + value = google_compute_network.main.name +} + +output "subnetwork" { + description = "The Cloud Composer Subnetwork." + value = google_compute_subnetwork.main.name +} + +output "composer_env_id" { + description = "ID of Cloud Composer Environment." + value = module.simple-composer.composer_env_id +} + +output "gcs_bucket" { + description = "Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment." + value = module.simple-composer.gcs_bucket +} + +output "airflow_uri" { + description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment." + value = module.simple-composer.airflow_uri +} diff --git a/test/fixtures/simple-composer-env-v3/variables.tf b/test/fixtures/simple-composer-env-v3/variables.tf new file mode 100644 index 00000000..e7b5f755 --- /dev/null +++ b/test/fixtures/simple-composer-env-v3/variables.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project ID where Cloud Composer Environment is created." + type = string +} + +variable "composer_sa" { + description = "Service Account to be used for running Cloud Composer Environment." + type = string +} + +variable "region" { + description = "Region where Cloud Composer Environment is created." + type = string + default = "us-central1" +} From 54a51b71336263049ca7013260b5de28becd823e Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Sat, 1 Feb 2025 04:53:26 +0000 Subject: [PATCH 05/20] chore: added test fixtures --- test/fixtures/simple-composer-env-v3/main.tf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/fixtures/simple-composer-env-v3/main.tf b/test/fixtures/simple-composer-env-v3/main.tf index 6edc98bc..7f88c6fc 100644 --- a/test/fixtures/simple-composer-env-v3/main.tf +++ b/test/fixtures/simple-composer-env-v3/main.tf @@ -17,13 +17,13 @@ module "simple-composer" { source = "../../../examples/simple_composer_env_v3" - project_id = var.project_id - composer_env_name = "composer-env-${random_id.random_suffix.hex}" - region = var.region - composer_service_account = var.composer_sa - network = google_compute_network.main.name - subnetwork = google_compute_subnetwork.main.name - create_network_attachment = true + project_id = var.project_id + composer_env_name = "composer-env-${random_id.random_suffix.hex}" + region = var.region + composer_service_account = var.composer_sa + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + create_network_attachment = true } resource "random_id" "random_suffix" { From 62d04966f6811be6d67091c5029a884761868a7c Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Sat, 1 Feb 2025 05:24:13 +0000 Subject: [PATCH 06/20] chore: added test fixtures --- test/fixtures/simple-composer-env-v3/main.tf | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/fixtures/simple-composer-env-v3/main.tf b/test/fixtures/simple-composer-env-v3/main.tf index 7f88c6fc..aa2e4436 100644 --- a/test/fixtures/simple-composer-env-v3/main.tf +++ b/test/fixtures/simple-composer-env-v3/main.tf @@ -17,13 +17,12 @@ module "simple-composer" { source = "../../../examples/simple_composer_env_v3" - project_id = var.project_id - composer_env_name = "composer-env-${random_id.random_suffix.hex}" - region = var.region - composer_service_account = var.composer_sa - network = google_compute_network.main.name - subnetwork = google_compute_subnetwork.main.name - create_network_attachment = true + project_id = var.project_id + composer_env_name = "composer-env-${random_id.random_suffix.hex}" + region = var.region + composer_service_account = var.composer_sa + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name } resource "random_id" "random_suffix" { From ecc7d30bd3cd00a2f3c9a44509f4c2d577da79ea Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Sat, 1 Feb 2025 06:07:53 +0000 Subject: [PATCH 07/20] chore: added test fixtures --- examples/simple_composer_env_v3/main.tf | 8 ++++---- modules/create_environment_v3/README.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/simple_composer_env_v3/main.tf b/examples/simple_composer_env_v3/main.tf index a3f2678a..874a000f 100644 --- a/examples/simple_composer_env_v3/main.tf +++ b/examples/simple_composer_env_v3/main.tf @@ -56,27 +56,27 @@ module "simple-composer-environment" { scheduler = { cpu = 0.5 - memory_gb = 1.875 + memory_gb = 1 storage_gb = 1 count = 2 } dag_processor = { cpu = 0.5 - memory_gb = 1.875 + memory_gb = 1 storage_gb = 1 count = 2 } web_server = { cpu = 0.5 - memory_gb = 1.875 + memory_gb = 1 storage_gb = 1 } worker = { cpu = 0.5 - memory_gb = 1.875 + memory_gb = 1 storage_gb = 1 min_count = 2 max_count = 3 diff --git a/modules/create_environment_v3/README.md b/modules/create_environment_v3/README.md index 81b8759b..d0412b59 100644 --- a/modules/create_environment_v3/README.md +++ b/modules/create_environment_v3/README.md @@ -24,27 +24,27 @@ module "simple-composer-environment" { scheduler = { cpu = 0.5 - memory_gb = 1.875 + memory_gb = 1 storage_gb = 1 count = 2 } dag_processor = { cpu = 0.5 - memory_gb = 1.875 + memory_gb = 1 storage_gb = 1 count = 2 } web_server = { cpu = 0.5 - memory_gb = 1.875 + memory_gb = 1 storage_gb = 1 } worker = { cpu = 0.5 - memory_gb = 1.875 + memory_gb = 1 storage_gb = 1 min_count = 2 max_count = 3 From ad06b7987efdcd01301e57b5862157aef6530c80 Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Fri, 7 Feb 2025 09:25:03 +0000 Subject: [PATCH 08/20] feat: fixed composer 3 env --- build/int.cloudbuild.yaml | 41 +++++++--------------- examples/simple_composer_env_v3/main.tf | 37 ++----------------- modules/create_environment_v3/README.md | 3 +- modules/create_environment_v3/main.tf | 1 + modules/create_environment_v3/variables.tf | 8 ++++- 5 files changed, 24 insertions(+), 66 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 4b1eb84d..c25c288e 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -1,4 +1,4 @@ -# Copyright 2020 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,22 +25,17 @@ steps: - 'TF_VAR_folder_id=$_FOLDER_ID' - 'TF_VAR_billing_account=$_BILLING_ACCOUNT' -- id: create all +- id: init-all waitFor: - prepare name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create'] + args: ['/bin/bash', '-c', 'cft test run all --stage init --verbose'] # ----- SUITE simple-composer-env-v2 -- id: init-simple-composer-env-v2 - waitFor: - - create all - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage init --verbose'] - id: apply-simple-composer-env-v2 waitFor: - - init-simple-composer-env-v2 + - init-all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage apply --verbose'] - id: verify-simple-composer-env-v2 @@ -54,17 +49,12 @@ steps: name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose'] - # ----- SUITE composer-v2-sharedvpc-prereq-local -- id: init-composer-v2-sharedvpc-prereq - waitFor: - - destroy-simple-composer-env-v2 - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage init --verbose'] - id: apply-composer-v2-sharedvpc-prereq waitFor: - - init-composer-v2-sharedvpc-prereq + - init-all + - destroy-simple-composer-env-v2 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage apply --verbose'] - id: verify-composer-v2-sharedvpc-prereq @@ -78,30 +68,23 @@ steps: name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage destroy --verbose'] - # ----- SUITE simple-composer-env-v3 -- id: init-simple-composer-env-v3 - waitFor: - - create all - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage init --verbose'] - id: apply-simple-composer-env-v3 waitFor: - - init-simple-composer-env-v3 + - init-all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage apply --verbose'] -- id: verify-simple-composer-env-v3 + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage apply --verbose'] +- id: verify-simple-composer-env-v2 waitFor: - apply-simple-composer-env-v3 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage verify --verbose'] -- id: destroy-simple-composer-env-v3 + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose'] +- id: destroy-simple-composer-env-v2 waitFor: - verify-simple-composer-env-v3 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage destroy --verbose'] - + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose'] tags: - 'ci' diff --git a/examples/simple_composer_env_v3/main.tf b/examples/simple_composer_env_v3/main.tf index 874a000f..180814d3 100644 --- a/examples/simple_composer_env_v3/main.tf +++ b/examples/simple_composer_env_v3/main.tf @@ -51,43 +51,10 @@ module "simple-composer-environment" { environment_size = "ENVIRONMENT_SIZE_SMALL" use_private_environment = true + enable_private_builds_only = true cloud_data_lineage_integration = true resilience_mode = "STANDARD_RESILIENCE" - scheduler = { - cpu = 0.5 - memory_gb = 1 - storage_gb = 1 - count = 2 - } - - dag_processor = { - cpu = 0.5 - memory_gb = 1 - storage_gb = 1 - count = 2 - } - - web_server = { - cpu = 0.5 - memory_gb = 1 - storage_gb = 1 - } - - worker = { - cpu = 0.5 - memory_gb = 1 - storage_gb = 1 - min_count = 2 - max_count = 3 - } - - triggerer = { - cpu = 1 - memory_gb = 1 - count = 2 - } - scheduled_snapshots_config = { enabled = true snapshot_location = google_storage_bucket.my_bucket.url @@ -96,7 +63,7 @@ module "simple-composer-environment" { } maintenance_start_time = "2025-02-01T00:00:00Z" - maintenance_end_time = "2025-03-01T12:00:00Z" + maintenance_end_time = "2025-05-01T12:00:00Z" maintenance_recurrence = "FREQ=WEEKLY;BYDAY=SU,SA" depends_on = [ diff --git a/modules/create_environment_v3/README.md b/modules/create_environment_v3/README.md index d0412b59..a4e8e751 100644 --- a/modules/create_environment_v3/README.md +++ b/modules/create_environment_v3/README.md @@ -70,10 +70,11 @@ module "simple-composer-environment" { | composer\_service\_account | Service Account for running Cloud Composer. | `string` | `null` | no | | create\_network\_attachment | Either create a new network attachment or use existing one. If true, provide the subnet details. | `bool` | `true` | no | | dag\_processor | Configuration for resources used by Airflow workers. | 
object({
    cpu        = string
    memory_gb  = number
    storage_gb = number
    count      = number
  })
| 
{
  "count": 2,
  "cpu": 2,
  "memory_gb": 7.5,
  "storage_gb": 5
}
| no | +| enable\_private\_builds\_only | If true, builds performed during operations that install Python packages have only private connectivity to Google services. If false, the builds also have access to the internet. | `bool` | `false` | no | | env\_variables | Variables of the airflow environment. | `map(string)` | `{}` | no | | environment\_size | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. Values for environment size are: `ENVIRONMENT_SIZE_SMALL`, `ENVIRONMENT_SIZE_MEDIUM`, and `ENVIRONMENT_SIZE_LARGE`. | `string` | `"ENVIRONMENT_SIZE_MEDIUM"` | no | | grant\_sa\_agent\_permission | Cloud Composer relies on Workload Identity as Google API authentication mechanism for Airflow. | `bool` | `true` | no | -| image\_version | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-3-airflow-2.10.2-build.5"` | no | +| image\_version | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-3-airflow-2.10.2-build.7"` | no | | kms\_key\_name | Customer-managed Encryption Key fully qualified resource name, i.e. projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key. | `string` | `null` | no | | labels | The resource labels (a map of key/value pairs) to be applied to the Cloud Composer. | `map(string)` | `{}` | no | | maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `null` | no | diff --git a/modules/create_environment_v3/main.tf b/modules/create_environment_v3/main.tf index e2700a01..b01f1c82 100644 --- a/modules/create_environment_v3/main.tf +++ b/modules/create_environment_v3/main.tf @@ -38,6 +38,7 @@ resource "google_composer_environment" "composer_env" { config { enable_private_environment = var.use_private_environment # reusing the existing variable name from previous versions + enable_private_builds_only = var.enable_private_builds_only environment_size = var.environment_size resilience_mode = var.resilience_mode diff --git a/modules/create_environment_v3/variables.tf b/modules/create_environment_v3/variables.tf index 26652b2c..6f72e6bd 100644 --- a/modules/create_environment_v3/variables.tf +++ b/modules/create_environment_v3/variables.tf @@ -97,7 +97,7 @@ variable "env_variables" { variable "image_version" { type = string description = "The version of the aiflow running in the cloud composer environment." - default = "composer-3-airflow-2.10.2-build.5" + default = "composer-3-airflow-2.10.2-build.7" } variable "web_server_plugins_mode" { @@ -118,6 +118,12 @@ variable "use_private_environment" { default = false } +variable "enable_private_builds_only" { + type = bool + description = "If true, builds performed during operations that install Python packages have only private connectivity to Google services. If false, the builds also have access to the internet." + default = false +} + variable "maintenance_start_time" { description = "Time window specified for daily or recurring maintenance operations in RFC3339 format" type = string From 7e21f44eb708d69a32441e805d5adc8a824a030c Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Fri, 7 Feb 2025 09:28:10 +0000 Subject: [PATCH 09/20] chore: fix build --- build/int.cloudbuild.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index c25c288e..23acc7a8 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -75,12 +75,12 @@ steps: - init-all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage apply --verbose'] -- id: verify-simple-composer-env-v2 +- id: verify-simple-composer-env-v3 waitFor: - apply-simple-composer-env-v3 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose'] -- id: destroy-simple-composer-env-v2 +- id: destroy-simple-composer-env-v3 waitFor: - verify-simple-composer-env-v3 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' From ba7eb9c60dd0c9853a8f41530358d91d0414c482 Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Fri, 7 Feb 2025 10:06:32 +0000 Subject: [PATCH 10/20] chore: fix build --- build/int.cloudbuild.yaml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 23acc7a8..19468a2d 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -43,11 +43,6 @@ steps: - apply-simple-composer-env-v2 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose'] -- id: destroy-simple-composer-env-v2 - waitFor: - - verify-simple-composer-env-v2 - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose'] # ----- SUITE composer-v2-sharedvpc-prereq-local @@ -62,11 +57,6 @@ steps: - apply-composer-v2-sharedvpc-prereq name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage verify --verbose'] -- id: destroy-composer-v2-sharedvpc-prereq - waitFor: - - verify-composer-v2-sharedvpc-prereq - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage destroy --verbose'] # ----- SUITE simple-composer-env-v3 @@ -80,11 +70,15 @@ steps: - apply-simple-composer-env-v3 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose'] -- id: destroy-simple-composer-env-v3 + + +- id: destroy-all waitFor: + - verify-simple-composer-env-v2 + - verify-composer-v2-sharedvpc-prereq - verify-simple-composer-env-v3 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose'] + args: ['/bin/bash', '-c', 'cft test run all --stage destroy --verbose'] tags: - 'ci' From f2bba0e5815adc19a449be0650aa5e26222de2ae Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Fri, 7 Feb 2025 10:13:43 +0000 Subject: [PATCH 11/20] chore: fix build --- build/int.cloudbuild.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 19468a2d..1a10b753 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -49,7 +49,6 @@ steps: - id: apply-composer-v2-sharedvpc-prereq waitFor: - init-all - - destroy-simple-composer-env-v2 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage apply --verbose'] - id: verify-composer-v2-sharedvpc-prereq From 508958f69fc1128520889716d459d466525e1c39 Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Fri, 7 Feb 2025 11:02:28 +0000 Subject: [PATCH 12/20] chore: fix build --- build/int.cloudbuild.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 1a10b753..8754dcac 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -63,12 +63,12 @@ steps: waitFor: - init-all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage apply --verbose'] + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage apply --verbose'] - id: verify-simple-composer-env-v3 waitFor: - apply-simple-composer-env-v3 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose'] + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage verify --verbose'] - id: destroy-all From a44706db7851870e9a5915c16f3423efe6a45ef1 Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Fri, 7 Feb 2025 13:24:36 +0000 Subject: [PATCH 13/20] chore: fix build --- build/int.cloudbuild.yaml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 8754dcac..dadf50cf 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -70,14 +70,23 @@ steps: name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage verify --verbose'] - -- id: destroy-all +- id: destroy-v2 waitFor: - verify-simple-composer-env-v2 + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose'] + +- id: destroy-v2-sharedvpc-prereq + waitFor: - verify-composer-v2-sharedvpc-prereq + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage verify --verbose'] + +- id: destroy-v3 + waitFor: - verify-simple-composer-env-v3 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run all --stage destroy --verbose'] + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage verify --verbose'] tags: - 'ci' From e86345c4987eb3e43251cc8a02db8af7167a3529 Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Sun, 9 Feb 2025 17:22:24 +0000 Subject: [PATCH 14/20] chore: readded kitchen tests --- build/int.cloudbuild.yaml | 8 ++- test/fixtures/simple-composer-env-v3/main.tf | 30 ----------- .../simple-composer-env-v3/network.tf | 46 ----------------- .../simple-composer-env-v3/outputs.tf | 50 ------------------- .../simple-composer-env-v3/variables.tf | 31 ------------ 5 files changed, 7 insertions(+), 158 deletions(-) delete mode 100644 test/fixtures/simple-composer-env-v3/main.tf delete mode 100644 test/fixtures/simple-composer-env-v3/network.tf delete mode 100644 test/fixtures/simple-composer-env-v3/outputs.tf delete mode 100644 test/fixtures/simple-composer-env-v3/variables.tf diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index dadf50cf..d3e6ffe7 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -25,10 +25,16 @@ steps: - 'TF_VAR_folder_id=$_FOLDER_ID' - 'TF_VAR_billing_account=$_BILLING_ACCOUNT' -- id: init-all +- id: create all waitFor: - prepare name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create'] + +- id: init-all + waitFor: + - create all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run all --stage init --verbose'] # ----- SUITE simple-composer-env-v2 diff --git a/test/fixtures/simple-composer-env-v3/main.tf b/test/fixtures/simple-composer-env-v3/main.tf deleted file mode 100644 index aa2e4436..00000000 --- a/test/fixtures/simple-composer-env-v3/main.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -module "simple-composer" { - source = "../../../examples/simple_composer_env_v3" - - project_id = var.project_id - composer_env_name = "composer-env-${random_id.random_suffix.hex}" - region = var.region - composer_service_account = var.composer_sa - network = google_compute_network.main.name - subnetwork = google_compute_subnetwork.main.name -} - -resource "random_id" "random_suffix" { - byte_length = 2 -} diff --git a/test/fixtures/simple-composer-env-v3/network.tf b/test/fixtures/simple-composer-env-v3/network.tf deleted file mode 100644 index 7a236a64..00000000 --- a/test/fixtures/simple-composer-env-v3/network.tf +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "google_compute_network" "main" { - project = var.project_id - name = "ci-composer-test-${random_string.suffix.result}" - auto_create_subnetworks = false -} - -resource "google_compute_subnetwork" "main" { - project = var.project_id - name = "ci-composer-test-${random_string.suffix.result}" - ip_cidr_range = "10.0.0.0/17" - region = var.region - network = google_compute_network.main.self_link - private_ip_google_access = true - - secondary_ip_range { - range_name = "ci-composer-test-pods-${random_string.suffix.result}" - ip_cidr_range = "192.168.0.0/18" - } - - secondary_ip_range { - range_name = "ci-composer-test-services-${random_string.suffix.result}" - ip_cidr_range = "192.168.64.0/18" - } -} - -resource "random_string" "suffix" { - length = 4 - special = false - upper = false -} diff --git a/test/fixtures/simple-composer-env-v3/outputs.tf b/test/fixtures/simple-composer-env-v3/outputs.tf deleted file mode 100644 index fde1fa10..00000000 --- a/test/fixtures/simple-composer-env-v3/outputs.tf +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "project_id" { - description = "Project ID where Cloud Composer Environment is created." - value = var.project_id -} - -output "composer_env_name" { - description = "Name of the Cloud Composer Environment." - value = module.simple-composer.composer_env_name -} - -output "network" { - description = "The Cloud Composer Network." - value = google_compute_network.main.name -} - -output "subnetwork" { - description = "The Cloud Composer Subnetwork." - value = google_compute_subnetwork.main.name -} - -output "composer_env_id" { - description = "ID of Cloud Composer Environment." - value = module.simple-composer.composer_env_id -} - -output "gcs_bucket" { - description = "Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment." - value = module.simple-composer.gcs_bucket -} - -output "airflow_uri" { - description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment." - value = module.simple-composer.airflow_uri -} diff --git a/test/fixtures/simple-composer-env-v3/variables.tf b/test/fixtures/simple-composer-env-v3/variables.tf deleted file mode 100644 index e7b5f755..00000000 --- a/test/fixtures/simple-composer-env-v3/variables.tf +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project ID where Cloud Composer Environment is created." - type = string -} - -variable "composer_sa" { - description = "Service Account to be used for running Cloud Composer Environment." - type = string -} - -variable "region" { - description = "Region where Cloud Composer Environment is created." - type = string - default = "us-central1" -} From 506637acc173ab53f21a15f5c7bfc6d583f214b2 Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Mon, 10 Feb 2025 17:59:33 +0000 Subject: [PATCH 15/20] chore: removed tests --- build/int.cloudbuild.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index d3e6ffe7..dadf50cf 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -25,15 +25,9 @@ steps: - 'TF_VAR_folder_id=$_FOLDER_ID' - 'TF_VAR_billing_account=$_BILLING_ACCOUNT' -- id: create all - waitFor: - - prepare - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create'] - - id: init-all waitFor: - - create all + - prepare name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run all --stage init --verbose'] From bdc933f566204360388af81e7350e632bc36bb31 Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Tue, 11 Feb 2025 15:32:38 +0000 Subject: [PATCH 16/20] feat: updated folder name --- .../simple_composer_env_v3_test.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/integration/{simple-composer-env-v3 => simple_composer_env_v3}/simple_composer_env_v3_test.go (100%) diff --git a/test/integration/simple-composer-env-v3/simple_composer_env_v3_test.go b/test/integration/simple_composer_env_v3/simple_composer_env_v3_test.go similarity index 100% rename from test/integration/simple-composer-env-v3/simple_composer_env_v3_test.go rename to test/integration/simple_composer_env_v3/simple_composer_env_v3_test.go From 219696a88f8fe23c98086af454995301cd2cd80d Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Tue, 11 Feb 2025 16:03:09 +0000 Subject: [PATCH 17/20] chore: variables corrected --- examples/simple_composer_env_v3/README.md | 6 ++-- examples/simple_composer_env_v3/main.tf | 6 ++-- examples/simple_composer_env_v3/network.tf | 36 ++++++++++++++++++++ examples/simple_composer_env_v3/variables.tf | 13 ++----- 4 files changed, 43 insertions(+), 18 deletions(-) create mode 100644 examples/simple_composer_env_v3/network.tf diff --git a/examples/simple_composer_env_v3/README.md b/examples/simple_composer_env_v3/README.md index c7465e70..3fceab02 100644 --- a/examples/simple_composer_env_v3/README.md +++ b/examples/simple_composer_env_v3/README.md @@ -10,11 +10,9 @@ This example also creates a Cloud Storage Bucket for scheduled snapshots and ass | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | composer\_env\_name | Name of Cloud Composer Environment. | `string` | `"ci-composer"` | no | -| composer\_service\_account | Service Account to be used for running Cloud Composer Environment. | `string` | n/a | yes | -| network | Network where Cloud Composer is created. | `string` | n/a | yes | +| composer\_sa | Service Account to be used for running Cloud Composer Environment. | `string` | n/a | yes | | project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes | -| region | Region where Cloud Composer Environment is created. | `string` | n/a | yes | -| subnetwork | Name of the Subetwork where Cloud Composer is created. | `string` | n/a | yes | +| region | Region where Cloud Composer Environment is created. | `string` | `"us-central1"` | no | ## Outputs diff --git a/examples/simple_composer_env_v3/main.tf b/examples/simple_composer_env_v3/main.tf index 180814d3..3b3e2a94 100644 --- a/examples/simple_composer_env_v3/main.tf +++ b/examples/simple_composer_env_v3/main.tf @@ -42,9 +42,9 @@ module "simple-composer-environment" { project_id = var.project_id composer_env_name = var.composer_env_name region = var.region - composer_service_account = var.composer_service_account - network = var.network - subnetwork = var.subnetwork + composer_service_account = var.composer_sa + network = google_compute_network.main.id + subnetwork = google_compute_subnetwork.main.id create_network_attachment = true grant_sa_agent_permission = false diff --git a/examples/simple_composer_env_v3/network.tf b/examples/simple_composer_env_v3/network.tf new file mode 100644 index 00000000..c911d635 --- /dev/null +++ b/examples/simple_composer_env_v3/network.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_network" "main" { + project = var.project_id + name = "ci-composer-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + project = var.project_id + name = "ci-composer-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + private_ip_google_access = true +} + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} diff --git a/examples/simple_composer_env_v3/variables.tf b/examples/simple_composer_env_v3/variables.tf index 4ef8fd66..f8138770 100644 --- a/examples/simple_composer_env_v3/variables.tf +++ b/examples/simple_composer_env_v3/variables.tf @@ -28,19 +28,10 @@ variable "composer_env_name" { variable "region" { description = "Region where Cloud Composer Environment is created." type = string + default = "us-central1" } -variable "composer_service_account" { +variable "composer_sa" { description = "Service Account to be used for running Cloud Composer Environment." type = string } - -variable "network" { - description = "Network where Cloud Composer is created." - type = string -} - -variable "subnetwork" { - description = "Name of the Subetwork where Cloud Composer is created." - type = string -} From b6ccd2d845ede99ccf0d9b6c52b75fe3b45b0b91 Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Tue, 11 Feb 2025 16:23:23 +0000 Subject: [PATCH 18/20] chore: variables corrected --- examples/simple_composer_env_v3/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/simple_composer_env_v3/main.tf b/examples/simple_composer_env_v3/main.tf index 3b3e2a94..da5ea930 100644 --- a/examples/simple_composer_env_v3/main.tf +++ b/examples/simple_composer_env_v3/main.tf @@ -32,7 +32,7 @@ resource "google_storage_bucket" "my_bucket" { resource "google_storage_bucket_iam_member" "object_admin" { bucket = google_storage_bucket.my_bucket.name role = "roles/storage.objectAdmin" - member = "serviceAccount:${var.composer_service_account}" + member = "serviceAccount:${var.composer_sa}" } module "simple-composer-environment" { From 8c5cd6c7d4e88c70ee920ca1ea41c15b5ad5b45c Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Tue, 11 Feb 2025 17:03:28 +0000 Subject: [PATCH 19/20] chore: variables corrected --- examples/simple_composer_env_v3/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/simple_composer_env_v3/main.tf b/examples/simple_composer_env_v3/main.tf index da5ea930..c7c08cdf 100644 --- a/examples/simple_composer_env_v3/main.tf +++ b/examples/simple_composer_env_v3/main.tf @@ -43,8 +43,8 @@ module "simple-composer-environment" { composer_env_name = var.composer_env_name region = var.region composer_service_account = var.composer_sa - network = google_compute_network.main.id - subnetwork = google_compute_subnetwork.main.id + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name create_network_attachment = true grant_sa_agent_permission = false From 5d8c9cc38c1a36ad3625c6ec42b7e5627e7a4341 Mon Sep 17 00:00:00 2001 From: prabhu34 <18209477+prabhu34@users.noreply.github.com> Date: Tue, 11 Feb 2025 17:53:30 +0000 Subject: [PATCH 20/20] chore: outputs corrected --- examples/simple_composer_env_v3/README.md | 1 + examples/simple_composer_env_v3/outputs.tf | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/examples/simple_composer_env_v3/README.md b/examples/simple_composer_env_v3/README.md index 3fceab02..d4b35fd4 100644 --- a/examples/simple_composer_env_v3/README.md +++ b/examples/simple_composer_env_v3/README.md @@ -22,6 +22,7 @@ This example also creates a Cloud Storage Bucket for scheduled snapshots and ass | composer\_env\_id | ID of Cloud Composer Environment. | | composer\_env\_name | Name of the Cloud Composer Environment. | | gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. | +| project\_id | Project ID where Cloud Composer Environment is created. | diff --git a/examples/simple_composer_env_v3/outputs.tf b/examples/simple_composer_env_v3/outputs.tf index 7a24325b..78223c33 100644 --- a/examples/simple_composer_env_v3/outputs.tf +++ b/examples/simple_composer_env_v3/outputs.tf @@ -14,6 +14,11 @@ * limitations under the License. */ +output "project_id" { + description = "Project ID where Cloud Composer Environment is created." + value = var.project_id +} + output "composer_env_name" { description = "Name of the Cloud Composer Environment." value = module.simple-composer-environment.composer_env_name