terraform build.

Author: sandonjacobs

.gitignore

@@ -19,30 +19,12 @@ replay_pid*
 !.vscode/*.code-snippets
 .history/
 *.vsix
-.idea/**/workspace.xml
-.idea/**/tasks.xml
-.idea/**/usage.statistics.xml
-.idea/**/dictionaries
-.idea/**/shelf
-.idea/**/aws.xml
-.idea/**/contentModel.xml
-.idea/**/dataSources/
-.idea/**/dataSources.ids
-.idea/**/dataSources.local.xml
-.idea/**/sqlDataSources.xml
-.idea/**/dynamic.xml
-.idea/**/uiDesigner.xml
-.idea/**/dbnavigator.xml
-.idea/**/gradle.xml
-.idea/**/libraries
+.idea/
 cmake-build-*/
-.idea/**/mongoSettings.xml
 *.iws
 out/
 .idea_modules/
 atlassian-ide-plugin.xml
-.idea/replstate.xml
-.idea/sonarlint/
 com_crashlytics_export_strings.xml
 crashlytics.properties
 crashlytics-build.properties

common/utils/src/main/resources/.gitignore

@@ -0,0 +1 @@
+cloud.properties

cloud.properties.example …/main/resources/cloud.properties.examplecloud.properties.example renamed to common/utils/src/main/resources/cloud.properties.example cloud.properties.example renamed to common/utils/src/main/resources/cloud.properties.example

@@ -17,17 +17,20 @@ export CONFLUENT_CLOUD_API_KEY=<API KEY>
 export CONFLUENT_CLOUD_API_SECRET=<API SECRET>
 ```
 
-
 == Execute Terraform Manifests
 
 The terraform manifests require the Confluent Cloud organization ID in order to provision infrastructure. 
 This can be found in the Confluent Cloud console in the "Organization Settings" and exported to an environment variable:
 
+image::org-id.jpg[]
+
+image::org-id-2.jpg[]
+
 ```bash
 export TF_VAR_org_id=<ORG ID VALUE FROM CONSOLE>
 ```
 
-This value can also be queried by using the Confluent CLI to query your account, piping the resulting to a `jq` query:
+This value can also be queried by using the Confluent CLI to query your account, piping the result to a `jq` query and using that value in a `TF_VAR_`:
 
 ```bash
 export TF_VAR_org_id=$(confluent organization list -o json | jq -c -r '.[] | select(.is_current)' | jq '.id')
@@ -51,7 +54,7 @@ This `terraform output` command will create a file with those parameters:
 ```bash
 terraform output -json \
  | jq -r 'to_entries | map( {key: .key|tostring|split("_")|join("."), value: .value} ) | map("client.\(.key)=\(.value.value)") | .[]' \
- | while read -r line ; do echo "$line"; done > ../cloud.properties
+ | while read -r line ; do echo "$line"; done > ../common/utils/src/main/resources/cloud.properties
 ```
 
 == Teardown

terraform/README.adoc docs/TERRAFORM.adocterraform/README.adoc renamed to docs/TERRAFORM.adoc

@@ -1,79 +1,88 @@
-resource "confluent_flink_compute_pool" "main_flink_pool" {
-  display_name = "main_flink_pool"
-  cloud        = var.cloud_provider
-  region       = var.cloud_region
-  max_cfu      = 5
+resource "confluent_flink_compute_pool" "compute_pool_1" {
+  display_name     = "-workshop_compute_pool_1"
+  cloud            = var.cloud_provider
+  region           = var.cloud_region
+  max_cfu          = 10
   environment {
     id = confluent_environment.cc_env.id
   }
-}
 
-data "confluent_flink_region" "main_flink_region" {
-  cloud  = var.cloud_provider
-  region = var.cloud_region
+  lifecycle {
+    prevent_destroy = false
+  }
 }
 
-resource "confluent_service_account" "flink_developer" {
-  display_name = "${var.cc_env_name}-flink_developer"
-  description  = "Service account for flink developer"
+// Service account to perform a task within Confluent Cloud, such as executing a Flink statement
+resource "confluent_service_account" "statements-runner" {
+  display_name = "${var.cc_env_name}-statements-runner"
+  description  = "Service account for running Flink Statements in 'inventory' Kafka cluster"
+
+  lifecycle {
+    prevent_destroy = false
+  }
 }
 
-resource "confluent_role_binding" "fd_flink_developer" {
-  principal   = "User:${confluent_service_account.flink_developer.id}"
-  role_name   = "FlinkDeveloper"
+resource "confluent_role_binding" "statements-runner-environment-admin" {
+  principal   = "User:${confluent_service_account.statements-runner.id}"
+  role_name   = "EnvironmentAdmin"
   crn_pattern = confluent_environment.cc_env.resource_name
-
-  depends_on = [confluent_flink_compute_pool.main_flink_pool]
+  lifecycle {
+    prevent_destroy = false
+  }
 }
 
-resource "confluent_role_binding" "fd_kafka_write" {
-  principal   = "User:${confluent_service_account.flink_developer.id}"
-  role_name   = "DeveloperWrite"
-  crn_pattern = "${confluent_kafka_cluster.kafka_cluster.rbac_crn}/kafka=${confluent_kafka_cluster.kafka_cluster.id}/topic=*"
-
-  depends_on = [confluent_kafka_cluster.kafka_cluster]
+// https://docs.confluent.io/cloud/current/access-management/access-control/rbac/predefined-rbac-roles.html#flinkadmin
+resource "confluent_role_binding" "app-manager-flink-developer" {
+  principal   = "User:${confluent_service_account.app-manager.id}"
+  role_name   = "FlinkAdmin"
+  crn_pattern = confluent_environment.cc_env.resource_name
+  lifecycle {
+    prevent_destroy = false
+  }
 }
 
-resource "confluent_role_binding" "fd_kafka_read" {
-  principal   = "User:${confluent_service_account.flink_developer.id}"
-  role_name   = "DeveloperRead"
-  crn_pattern = "${confluent_kafka_cluster.kafka_cluster.rbac_crn}/kafka=${confluent_kafka_cluster.kafka_cluster.id}/topic=*"
+data "confluent_organization" "main" {}
 
-  depends_on = [confluent_kafka_cluster.kafka_cluster]
+// https://docs.confluent.io/cloud/current/access-management/access-control/rbac/predefined-rbac-roles.html#assigner
+// https://docs.confluent.io/cloud/current/flink/operate-and-deploy/flink-rbac.html#submit-long-running-statements
+resource "confluent_role_binding" "app-manager-assigner" {
+  principal   = "User:${confluent_service_account.app-manager.id}"
+  role_name   = "Assigner"
+  crn_pattern = "${data.confluent_organization.main.resource_name}/service-account=${confluent_service_account.statements-runner.id}"
+  lifecycle {
+    prevent_destroy = false
+  }
 }
 
-resource "confluent_role_binding" "fd_schema_registry_write" {
-  principal   = "User:${confluent_service_account.flink_developer.id}"
-  role_name   = "DeveloperWrite"
-  crn_pattern = "${data.confluent_schema_registry_cluster.advanced.resource_name}/subject=*"
+data "confluent_flink_region" "us-east-2" {
+  cloud  = var.cloud_provider
+  region = var.cloud_region
 }
 
-resource "confluent_role_binding" "fd_schema_registry_read" {
-  principal   = "User:${confluent_service_account.flink_developer.id}"
-  role_name   = "DeveloperRead"
-  crn_pattern = "${data.confluent_schema_registry_cluster.advanced.resource_name}/subject=*"
+data "confluent_flink_region" "main" {
+  cloud  = var.cloud_provider
+  region = var.cloud_region
 }
 
-resource "confluent_api_key" "flink_developer_api_key" {
-  display_name = "flink_developer_api_key"
-  description  = "Flink Developer API Key that is owned by 'flink_developer' service account"
+
+resource "confluent_api_key" "app-manager-flink-api-key" {
+  display_name = "app-manager-flink-api-key"
+  description  = "Flink API Key that is owned by 'app-manager' service account"
   owner {
-    id          = confluent_service_account.flink_developer.id
-    api_version = confluent_service_account.flink_developer.api_version
-    kind        = confluent_service_account.flink_developer.kind
+    id          = confluent_service_account.app-manager.id
+    api_version = confluent_service_account.app-manager.api_version
+    kind        = confluent_service_account.app-manager.kind
   }
-
   managed_resource {
-    id          = data.confluent_flink_region.main_flink_region.id
-    api_version = data.confluent_flink_region.main_flink_region.api_version
-    kind        = data.confluent_flink_region.main_flink_region.kind
-
+    id          = data.confluent_flink_region.us-east-2.id
+    api_version = confluent_flink_compute_pool.compute_pool_1.api_version
+    kind        = data.confluent_flink_region.us-east-2.kind
     environment {
       id = confluent_environment.cc_env.id
     }
   }
+  lifecycle {
+    prevent_destroy = false
+  }
+}
 
-  depends_on = [
-    confluent_service_account.flink_developer
-  ]
-}

docs/org-id-2.jpg

@@ -1,37 +1,45 @@
+# Update the config to use a cloud provider and region of your choice.
+# https://registry.terraform.io/providers/confluentinc/confluent/latest/docs/resources/confluent_kafka_cluster
 resource "confluent_kafka_cluster" "kafka_cluster" {
   display_name = var.cc_default_kafka_cluster_name
   availability = "SINGLE_ZONE"
   cloud        = var.cloud_provider
   region       = var.cloud_region
-  standard {}
+  basic {}
   environment {
     id = confluent_environment.cc_env.id
   }
 
   depends_on = [confluent_environment.cc_env]
 }
 
-# ---------------------------------------------------------------------------
-# API KEY and Role for Administration of Kafka
-# ---------------------------------------------------------------------------
-resource "confluent_service_account" "kafka_manager" {
-  display_name = "${var.cc_env_name}-kafka_manager"
+data "confluent_schema_registry_cluster" "advanced" {
+  environment {
+    id = confluent_environment.cc_env.id
+  }
+  depends_on = [confluent_kafka_cluster.kafka_cluster]
+}
+
+// 'app-manager' service account is required in this configuration to create 'purchase' topic and grant ACLs
+// to 'app-producer' and 'app-consumer' service accounts.
+resource "confluent_service_account" "app-manager" {
+  display_name = "${var.cc_default_kafka_cluster_name}-app-manager"
   description  = "Service account to manage Kafka cluster"
 }
 
-resource "confluent_role_binding" "kafka_manager_kafka_cluster_admin" {
-  principal   = "User:${confluent_service_account.kafka_manager.id}"
+resource "confluent_role_binding" "app-manager-kafka-cluster-admin" {
+  principal   = "User:${confluent_service_account.app-manager.id}"
   role_name   = "CloudClusterAdmin"
   crn_pattern = confluent_kafka_cluster.kafka_cluster.rbac_crn
 }
 
-resource "confluent_api_key" "kafka_manager_kafka_api_key" {
-  display_name = "kafka_manager_kafka_api_key"
-  description  = "Kafka API Key that is owned by 'kafka_manager' service account"
+resource "confluent_api_key" "app-manager-kafka-api-key" {
+  display_name = "app-manager-kafka-api-key"
+  description  = "Kafka API Key that is owned by 'app-manager' service account"
   owner {
-    id          = confluent_service_account.kafka_manager.id
-    api_version = confluent_service_account.kafka_manager.api_version
-    kind        = confluent_service_account.kafka_manager.kind
+    id          = confluent_service_account.app-manager.id
+    api_version = confluent_service_account.app-manager.api_version
+    kind        = confluent_service_account.app-manager.kind
   }
 
   managed_resource {
@@ -44,54 +52,76 @@ resource "confluent_api_key" "kafka_manager_kafka_api_key" {
     }
   }
 
+  # The goal is to ensure that confluent_role_binding.app-manager-kafka-cluster-admin is created before
+  # confluent_api_key.app-manager-kafka-api-key is used to create instances of
+  # confluent_kafka_topic, confluent_kafka_acl resources.
+
+  # 'depends_on' meta-argument is specified in confluent_api_key.app-manager-kafka-api-key to avoid having
+  # multiple copies of this definition in the configuration which would happen if we specify it in
+  # confluent_kafka_topic, confluent_kafka_acl resources instead.
   depends_on = [
-    confluent_environment.cc_env,
-    confluent_role_binding.kafka_manager_kafka_cluster_admin
+    confluent_role_binding.app-manager-kafka-cluster-admin
   ]
 }
 
-# ---------------------------------------------------------------------------
-# API KEY and Role for Developers on Kafka
-# ---------------------------------------------------------------------------
-
-resource "confluent_service_account" "kafka_developer" {
-  display_name = "${var.cc_env_name}-kafka_developer"
-  description  = "Service account for developer using Kafka cluster"
+resource "confluent_service_account" "env-manager" {
+  display_name = "${var.cc_default_kafka_cluster_name}-env-manager"
+  description  = "Service account to manage 'Staging' environment"
 }
 
-resource "confluent_role_binding" "kafka_developer_read_all_topics" {
-  principal   = "User:${confluent_service_account.kafka_manager.id}"
-  role_name   = "DeveloperRead"
-  crn_pattern = "${confluent_kafka_cluster.kafka_cluster.rbac_crn}/kafka=${confluent_kafka_cluster.kafka_cluster.id}/topic=*"
+resource "confluent_role_binding" "env-manager-environment-admin" {
+  principal   = "User:${confluent_service_account.env-manager.id}"
+  role_name   = "EnvironmentAdmin"
+  crn_pattern = confluent_environment.cc_env.resource_name
 }
 
-resource "confluent_role_binding" "kafka_developer_write_all_topics" {
-  principal   = "User:${confluent_service_account.kafka_manager.id}"
-  role_name   = "DeveloperWrite"
-  crn_pattern = "${confluent_kafka_cluster.kafka_cluster.rbac_crn}/kafka=${confluent_kafka_cluster.kafka_cluster.id}/topic=*"
-}
-
-resource "confluent_api_key" "kafka_developer_kafka_api_key" {
-  display_name = "kafka_developer_kafka_api_key"
-  description  = "Kafka API Key that is owned by 'kafka_developer' service account"
+resource "confluent_api_key" "env-manager-schema-registry-api-key" {
+  display_name = "env-manager-schema-registry-api-key"
+  description  = "Schema Registry API Key that is owned by 'env-manager' service account"
   owner {
-    id          = confluent_service_account.kafka_developer.id
-    api_version = confluent_service_account.kafka_developer.api_version
-    kind        = confluent_service_account.kafka_developer.kind
+    id          = confluent_service_account.env-manager.id
+    api_version = confluent_service_account.env-manager.api_version
+    kind        = confluent_service_account.env-manager.kind
   }
 
   managed_resource {
-    id          = confluent_kafka_cluster.kafka_cluster.id
-    api_version = confluent_kafka_cluster.kafka_cluster.api_version
-    kind        = confluent_kafka_cluster.kafka_cluster.kind
+    id          = data.confluent_schema_registry_cluster.advanced.id
+    api_version = data.confluent_schema_registry_cluster.advanced.api_version
+    kind        = data.confluent_schema_registry_cluster.advanced.kind
 
     environment {
       id = confluent_environment.cc_env.id
     }
   }
 
+  # The goal is to ensure that confluent_role_binding.env-manager-environment-admin is created before
+  # confluent_api_key.env-manager-schema-registry-api-key is used to create instances of
+  # confluent_schema resources.
+
+  # 'depends_on' meta-argument is specified in confluent_api_key.env-manager-schema-registry-api-key to avoid having
+  # multiple copies of this definition in the configuration which would happen if we specify it in
+  # confluent_schema resources instead.
   depends_on = [
-    confluent_role_binding.kafka_developer_read_all_topics,
-    confluent_role_binding.kafka_developer_write_all_topics
+    confluent_role_binding.env-manager-environment-admin,
+    data.confluent_schema_registry_cluster.advanced
   ]
 }
+
+resource "confluent_schema_registry_cluster_config" "schema_registry_cluster_config" {
+  schema_registry_cluster {
+    id = data.confluent_schema_registry_cluster.advanced.id
+  }
+  rest_endpoint = data.confluent_schema_registry_cluster.advanced.rest_endpoint
+  compatibility_level = "BACKWARD"
+  credentials {
+    key    = confluent_api_key.env-manager-schema-registry-api-key.id
+    secret = confluent_api_key.env-manager-schema-registry-api-key.secret
+  }
+
+  depends_on = [data.confluent_schema_registry_cluster.advanced,
+    confluent_api_key.env-manager-schema-registry-api-key]
+
+  lifecycle {
+    prevent_destroy = false
+  }
+}