Merge branch 'main' of github.com:gAmUssA/flink-for-java-workshop

Author: gAmUssA

.gitignore

@@ -19,30 +19,12 @@ replay_pid*
 !.vscode/*.code-snippets
 .history/
 *.vsix
-.idea/**/workspace.xml
-.idea/**/tasks.xml
-.idea/**/usage.statistics.xml
-.idea/**/dictionaries
-.idea/**/shelf
-.idea/**/aws.xml
-.idea/**/contentModel.xml
-.idea/**/dataSources/
-.idea/**/dataSources.ids
-.idea/**/dataSources.local.xml
-.idea/**/sqlDataSources.xml
-.idea/**/dynamic.xml
-.idea/**/uiDesigner.xml
-.idea/**/dbnavigator.xml
-.idea/**/gradle.xml
-.idea/**/libraries
+.idea/
 cmake-build-*/
-.idea/**/mongoSettings.xml
 *.iws
 out/
 .idea_modules/
 atlassian-ide-plugin.xml
-.idea/replstate.xml
-.idea/sonarlint/
 com_crashlytics_export_strings.xml
 crashlytics.properties
 crashlytics-build.properties
@@ -62,3 +44,6 @@ gradle-app.setting
 cloud.properties
 .vscode
 bin
+
+
+gradlew.bat

common/utils/src/main/resources/.gitignore

@@ -0,0 +1 @@
+cloud.properties

cloud.properties.example …/main/resources/cloud.properties.examplecloud.properties.example renamed to common/utils/src/main/resources/cloud.properties.example cloud.properties.example renamed to common/utils/src/main/resources/cloud.properties.example

@@ -17,17 +17,20 @@ export CONFLUENT_CLOUD_API_KEY=<API KEY>
 export CONFLUENT_CLOUD_API_SECRET=<API SECRET>
 ```
 
-
 == Execute Terraform Manifests
 
 The terraform manifests require the Confluent Cloud organization ID in order to provision infrastructure. 
 This can be found in the Confluent Cloud console in the "Organization Settings" and exported to an environment variable:
 
+image::org-id.jpg[]
+
+image::org-id-2.jpg[]
+
 ```bash
 export TF_VAR_org_id=<ORG ID VALUE FROM CONSOLE>
 ```
 
-This value can also be queried by using the Confluent CLI to query your account, piping the resulting to a `jq` query:
+This value can also be queried by using the Confluent CLI to query your account, piping the result to a `jq` query and using that value in a `TF_VAR_`:
 
 ```bash
 export TF_VAR_org_id=$(confluent organization list -o json | jq -c -r '.[] | select(.is_current)' | jq '.id')
@@ -51,7 +54,7 @@ This `terraform output` command will create a file with those parameters:
 ```bash
 terraform output -json \
  | jq -r 'to_entries | map( {key: .key|tostring|split("_")|join("."), value: .value} ) | map("client.\(.key)=\(.value.value)") | .[]' \
- | while read -r line ; do echo "$line"; done > ../cloud.properties
+ | while read -r line ; do echo "$line"; done > ../common/utils/src/main/resources/cloud.properties
 ```
 
 == Teardown

terraform/README.adoc docs/TERRAFORM.adocterraform/README.adoc renamed to docs/TERRAFORM.adoc

@@ -31,7 +31,7 @@ dependencies {
 
     // Test dependencies
     testImplementation("org.apache.flink:flink-test-utils:$flinkVersion")
-    testImplementation(platform("org.junit:junit-bom:5.10.0"))
+    testImplementation(platform("org.junit:junit-bom:5.12.0"))
     testImplementation("org.junit.jupiter:junit-jupiter")
     testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine")
 }

docs/org-id-2.jpg

@@ -1,79 +1,88 @@
-resource "confluent_flink_compute_pool" "main_flink_pool" {
-  display_name = "main_flink_pool"
-  cloud        = var.cloud_provider
-  region       = var.cloud_region
-  max_cfu      = 5
+resource "confluent_flink_compute_pool" "compute_pool_1" {
+  display_name     = "-workshop_compute_pool_1"
+  cloud            = var.cloud_provider
+  region           = var.cloud_region
+  max_cfu          = 10
   environment {
     id = confluent_environment.cc_env.id
   }
-}
 
-data "confluent_flink_region" "main_flink_region" {
-  cloud  = var.cloud_provider
-  region = var.cloud_region
+  lifecycle {
+    prevent_destroy = false
+  }
 }
 
-resource "confluent_service_account" "flink_developer" {
-  display_name = "${var.cc_env_name}-flink_developer"
-  description  = "Service account for flink developer"
+// Service account to perform a task within Confluent Cloud, such as executing a Flink statement
+resource "confluent_service_account" "statements-runner" {
+  display_name = "${var.cc_env_name}-statements-runner"
+  description  = "Service account for running Flink Statements in 'inventory' Kafka cluster"
+
+  lifecycle {
+    prevent_destroy = false
+  }
 }
 
-resource "confluent_role_binding" "fd_flink_developer" {
-  principal   = "User:${confluent_service_account.flink_developer.id}"
-  role_name   = "FlinkDeveloper"
+resource "confluent_role_binding" "statements-runner-environment-admin" {
+  principal   = "User:${confluent_service_account.statements-runner.id}"
+  role_name   = "EnvironmentAdmin"
   crn_pattern = confluent_environment.cc_env.resource_name
-
-  depends_on = [confluent_flink_compute_pool.main_flink_pool]
+  lifecycle {
+    prevent_destroy = false
+  }
 }
 
-resource "confluent_role_binding" "fd_kafka_write" {
-  principal   = "User:${confluent_service_account.flink_developer.id}"
-  role_name   = "DeveloperWrite"
-  crn_pattern = "${confluent_kafka_cluster.kafka_cluster.rbac_crn}/kafka=${confluent_kafka_cluster.kafka_cluster.id}/topic=*"
-
-  depends_on = [confluent_kafka_cluster.kafka_cluster]
+// https://docs.confluent.io/cloud/current/access-management/access-control/rbac/predefined-rbac-roles.html#flinkadmin
+resource "confluent_role_binding" "app-manager-flink-developer" {
+  principal   = "User:${confluent_service_account.app-manager.id}"
+  role_name   = "FlinkAdmin"
+  crn_pattern = confluent_environment.cc_env.resource_name
+  lifecycle {
+    prevent_destroy = false
+  }
 }
 
-resource "confluent_role_binding" "fd_kafka_read" {
-  principal   = "User:${confluent_service_account.flink_developer.id}"
-  role_name   = "DeveloperRead"
-  crn_pattern = "${confluent_kafka_cluster.kafka_cluster.rbac_crn}/kafka=${confluent_kafka_cluster.kafka_cluster.id}/topic=*"
+data "confluent_organization" "main" {}
 
-  depends_on = [confluent_kafka_cluster.kafka_cluster]
+// https://docs.confluent.io/cloud/current/access-management/access-control/rbac/predefined-rbac-roles.html#assigner
+// https://docs.confluent.io/cloud/current/flink/operate-and-deploy/flink-rbac.html#submit-long-running-statements
+resource "confluent_role_binding" "app-manager-assigner" {
+  principal   = "User:${confluent_service_account.app-manager.id}"
+  role_name   = "Assigner"
+  crn_pattern = "${data.confluent_organization.main.resource_name}/service-account=${confluent_service_account.statements-runner.id}"
+  lifecycle {
+    prevent_destroy = false
+  }
 }
 
-resource "confluent_role_binding" "fd_schema_registry_write" {
-  principal   = "User:${confluent_service_account.flink_developer.id}"
-  role_name   = "DeveloperWrite"
-  crn_pattern = "${data.confluent_schema_registry_cluster.advanced.resource_name}/subject=*"
+data "confluent_flink_region" "us-east-2" {
+  cloud  = var.cloud_provider
+  region = var.cloud_region
 }
 
-resource "confluent_role_binding" "fd_schema_registry_read" {
-  principal   = "User:${confluent_service_account.flink_developer.id}"
-  role_name   = "DeveloperRead"
-  crn_pattern = "${data.confluent_schema_registry_cluster.advanced.resource_name}/subject=*"
+data "confluent_flink_region" "main" {
+  cloud  = var.cloud_provider
+  region = var.cloud_region
 }
 
-resource "confluent_api_key" "flink_developer_api_key" {
-  display_name = "flink_developer_api_key"
-  description  = "Flink Developer API Key that is owned by 'flink_developer' service account"
+
+resource "confluent_api_key" "app-manager-flink-api-key" {
+  display_name = "app-manager-flink-api-key"
+  description  = "Flink API Key that is owned by 'app-manager' service account"
   owner {
-    id          = confluent_service_account.flink_developer.id
-    api_version = confluent_service_account.flink_developer.api_version
-    kind        = confluent_service_account.flink_developer.kind
+    id          = confluent_service_account.app-manager.id
+    api_version = confluent_service_account.app-manager.api_version
+    kind        = confluent_service_account.app-manager.kind
   }
-
   managed_resource {
-    id          = data.confluent_flink_region.main_flink_region.id
-    api_version = data.confluent_flink_region.main_flink_region.api_version
-    kind        = data.confluent_flink_region.main_flink_region.kind
-
+    id          = data.confluent_flink_region.us-east-2.id
+    api_version = confluent_flink_compute_pool.compute_pool_1.api_version
+    kind        = data.confluent_flink_region.us-east-2.kind
     environment {
       id = confluent_environment.cc_env.id
     }
   }
+  lifecycle {
+    prevent_destroy = false
+  }
+}
 
-  depends_on = [
-    confluent_service_account.flink_developer
-  ]
-}