feat: Update EMR virtual cluster module to use latest functionality offered by provider

Author: bryantbiggs

examples/virtual-cluster/main.tf

@@ -43,8 +43,8 @@ locals {
 module "complete" {
   source = "../../modules/virtual-cluster"
 
-  eks_cluster_id    = module.eks.cluster_name
-  oidc_provider_arn = module.eks.oidc_provider_arn
+  eks_cluster_name      = module.eks.cluster_name
+  eks_oidc_provider_arn = module.eks.oidc_provider_arn
 
   name             = "emr-custom"
   create_namespace = true
@@ -67,8 +67,8 @@ module "complete" {
 module "default" {
   source = "../../modules/virtual-cluster"
 
-  eks_cluster_id    = module.eks.cluster_name
-  oidc_provider_arn = module.eks.oidc_provider_arn
+  eks_cluster_name      = module.eks.cluster_name
+  eks_oidc_provider_arn = module.eks.oidc_provider_arn
 
   s3_bucket_arns = [
     module.s3_bucket.s3_bucket_arn,
@@ -160,6 +160,9 @@ module "eks" {
 
   enable_cluster_creator_admin_permissions = true
 
+  # Required for now until https://github.com/aws/containers-roadmap/issues/2397
+  enable_irsa = true
+
   compute_config = {
     enabled    = true
     node_pools = ["general-purpose", "system"]

modules/virtual-cluster/README.md

@@ -12,6 +12,9 @@ See [`examples`](https://github.com/terraform-aws-modules/terraform-aws-emr/tree
 module "emr_virtual_cluster" {
   source = "terraform-aws-modules/emr/aws//modules/virtual-cluster"
 
+  eks_cluster_name      = "example"
+  eks_oidc_provider_arn = "arn:aws:iam::0123456789:oidc-provider/eks-example"
+
   name             = "emr-custom"
   create_namespace = true
   namespace        = "emr-custom"
@@ -41,6 +44,9 @@ module "emr_virtual_cluster" {
 module "emr_virtual_cluster" {
   source = "terraform-aws-modules/emr/aws//modules/virtual-cluster"
 
+  eks_cluster_name      = "example"
+  eks_oidc_provider_arn = "arn:aws:iam::0123456789:oidc-provider/eks-example"
+
   name      = "emr-default"
   namespace = "emr-default"
 
@@ -104,6 +110,7 @@ No modules.
 |------|-------------|------|---------|:--------:|
 | <a name="input_annotations"></a> [annotations](#input\_annotations) | A map of annotations to add to all Kubernetes resources | `map(string)` | `{}` | no |
 | <a name="input_cloudwatch_log_group_arn"></a> [cloudwatch\_log\_group\_arn](#input\_cloudwatch\_log\_group\_arn) | ARN of the log group to use for the cluster logs | `string` | `"arn:aws:logs:*:*:*"` | no |
+| <a name="input_cloudwatch_log_group_class"></a> [cloudwatch\_log\_group\_class](#input\_cloudwatch\_log\_group\_class) | Specified the log class of the log group. Possible values are: `STANDARD` or `INFREQUENT_ACCESS` | `string` | `null` | no |
 | <a name="input_cloudwatch_log_group_kms_key_id"></a> [cloudwatch\_log\_group\_kms\_key\_id](#input\_cloudwatch\_log\_group\_kms\_key\_id) | If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html) | `string` | `null` | no |
 | <a name="input_cloudwatch_log_group_name"></a> [cloudwatch\_log\_group\_name](#input\_cloudwatch\_log\_group\_name) | The name of the log group. If a name is not provided, the default name format used is: `/emr-on-eks-logs/emr-workload/<NAMESPACE>` | `string` | `null` | no |
 | <a name="input_cloudwatch_log_group_retention_in_days"></a> [cloudwatch\_log\_group\_retention\_in\_days](#input\_cloudwatch\_log\_group\_retention\_in\_days) | Number of days to retain log events. Default retention - 7 days | `number` | `7` | no |
@@ -114,7 +121,8 @@ No modules.
 | <a name="input_create_iam_role"></a> [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created for EMR on EKS job execution role | `bool` | `true` | no |
 | <a name="input_create_kubernetes_role"></a> [create\_kubernetes\_role](#input\_create\_kubernetes\_role) | Determines whether a Kubernetes role is created for EMR on EKS | `bool` | `true` | no |
 | <a name="input_create_namespace"></a> [create\_namespace](#input\_create\_namespace) | Determines whether a Kubernetes namespace is created for EMR on EKS | `bool` | `true` | no |
-| <a name="input_eks_cluster_id"></a> [eks\_cluster\_id](#input\_eks\_cluster\_id) | EKS cluster ID | `string` | `""` | no |
+| <a name="input_eks_cluster_name"></a> [eks\_cluster\_name](#input\_eks\_cluster\_name) | EKS cluster name | `string` | `""` | no |
+| <a name="input_eks_oidc_provider_arn"></a> [eks\_oidc\_provider\_arn](#input\_eks\_oidc\_provider\_arn) | OIDC provider ARN for the EKS cluster | `string` | `""` | no |
 | <a name="input_iam_role_additional_policies"></a> [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the job execution IAM role | `any` | `{}` | no |
 | <a name="input_iam_role_description"></a> [iam\_role\_description](#input\_iam\_role\_description) | Description of the job execution role | `string` | `null` | no |
 | <a name="input_iam_role_path"></a> [iam\_role\_path](#input\_iam\_role\_path) | Job execution IAM role path | `string` | `null` | no |
@@ -123,7 +131,7 @@ No modules.
 | <a name="input_labels"></a> [labels](#input\_labels) | A map of labels to add to all Kubernetes resources | `map(string)` | `{}` | no |
 | <a name="input_name"></a> [name](#input\_name) | Name of the EMR on EKS virtual cluster | `string` | `""` | no |
 | <a name="input_namespace"></a> [namespace](#input\_namespace) | Kubernetes namespace for EMR on EKS | `string` | `"emr-containers"` | no |
-| <a name="input_oidc_provider_arn"></a> [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | OIDC provider ARN for the EKS cluster | `string` | `""` | no |
+| <a name="input_region"></a> [region](#input\_region) | Region where the resource(s) will be managed. Defaults to the Region set in the provider configuration | `string` | `null` | no |
 | <a name="input_role_name"></a> [role\_name](#input\_role\_name) | Name to use on IAM role created for EMR on EKS job execution role as well as Kubernetes RBAC role | `string` | `null` | no |
 | <a name="input_s3_bucket_arns"></a> [s3\_bucket\_arns](#input\_s3\_bucket\_arns) | S3 bucket ARNs for EMR on EKS job execution role to list, get objects, and put objects | `list(string)` | `[]` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no |

modules/virtual-cluster/main.tf

@@ -1,7 +1,9 @@
-data "aws_caller_identity" "current" {}
+data "aws_caller_identity" "current" {
+  count = var.create ? 1 : 0
+}
 
 locals {
-  account_id = data.aws_caller_identity.current.account_id
+  account_id = try(data.aws_caller_identity.current[0].account_id, "")
 
   internal_role_name = try(coalesce(var.role_name, var.name), "")
 
@@ -19,10 +21,12 @@ locals {
 resource "aws_emrcontainers_virtual_cluster" "this" {
   count = var.create ? 1 : 0
 
+  region = var.region
+
   name = var.name
 
   container_provider {
-    id   = var.eks_cluster_id
+    id   = var.eks_cluster_name
     type = "EKS"
 
     info {
@@ -145,27 +149,30 @@ locals {
 data "aws_iam_policy_document" "assume" {
   count = local.create_iam_role ? 1 : 0
 
+
+  # IRSA is default and only supported authentication method for now until wildcard support is added
+  # to EKS pod identity https://github.com/aws/containers-roadmap/issues/2397
   statement {
     sid     = "IRSA"
     effect  = "Allow"
     actions = ["sts:AssumeRoleWithWebIdentity"]
 
     principals {
       type        = "Federated"
-      identifiers = [var.oidc_provider_arn]
+      identifiers = [var.eks_oidc_provider_arn]
     }
 
     condition {
       test     = "StringLike"
-      variable = "${replace(var.oidc_provider_arn, "/^(.*provider/)/", "")}:sub"
+      variable = "${replace(var.eks_oidc_provider_arn, "/^(.*provider/)/", "")}:sub"
       # Terraform lacks support for a base32 function and role names with prefixes are unknown so a wildcard is used
       values = ["system:serviceaccount:${local.namespace}:emr-containers-sa-*-*-${local.account_id}-*"]
     }
 
     # https://aws.amazon.com/premiumsupport/knowledge-center/eks-troubleshoot-oidc-and-irsa/?nc1=h_ls
     condition {
       test     = "StringEquals"
-      variable = "${replace(var.oidc_provider_arn, "/^(.*provider/)/", "")}:aud"
+      variable = "${replace(var.eks_oidc_provider_arn, "/^(.*provider/)/", "")}:aud"
       values   = ["sts.amazonaws.com"]
     }
   }
@@ -258,10 +265,13 @@ resource "aws_iam_role_policy_attachment" "additional" {
 resource "aws_cloudwatch_log_group" "this" {
   count = var.create && var.create_cloudwatch_log_group ? 1 : 0
 
+  region = var.region
+
   name              = var.cloudwatch_log_group_use_name_prefix ? null : local.cloudwatch_log_group_name
   name_prefix       = var.cloudwatch_log_group_use_name_prefix ? "${local.cloudwatch_log_group_name}-" : null
   retention_in_days = var.cloudwatch_log_group_retention_in_days
   kms_key_id        = var.cloudwatch_log_group_kms_key_id
+  log_group_class   = var.cloudwatch_log_group_class
   skip_destroy      = var.cloudwatch_log_group_skip_destroy
 
   tags = local.tags

modules/virtual-cluster/variables.tf

@@ -4,6 +4,12 @@ variable "create" {
   default     = true
 }
 
+variable "region" {
+  description = "Region where the resource(s) will be managed. Defaults to the Region set in the provider configuration"
+  type        = string
+  default     = null
+}
+
 variable "tags" {
   description = "A map of tags to add to all resources"
   type        = map(string)
@@ -32,8 +38,8 @@ variable "name" {
   default     = ""
 }
 
-variable "eks_cluster_id" {
-  description = "EKS cluster ID"
+variable "eks_cluster_name" {
+  description = "EKS cluster name"
   type        = string
   default     = ""
 }
@@ -70,7 +76,7 @@ variable "create_iam_role" {
   default     = true
 }
 
-variable "oidc_provider_arn" {
+variable "eks_oidc_provider_arn" {
   description = "OIDC provider ARN for the EKS cluster"
   type        = string
   default     = ""
@@ -114,7 +120,7 @@ variable "iam_role_permissions_boundary" {
 
 variable "iam_role_additional_policies" {
   description = "Additional policies to be added to the job execution IAM role"
-  type        = any
+  type        = map(string)
   default     = {}
 }
 
@@ -146,6 +152,12 @@ variable "cloudwatch_log_group_kms_key_id" {
   default     = null
 }
 
+variable "cloudwatch_log_group_class" {
+  description = "Specified the log class of the log group. Possible values are: `STANDARD` or `INFREQUENT_ACCESS`"
+  type        = string
+  default     = null
+}
+
 variable "cloudwatch_log_group_name" {
   description = "The name of the log group. If a name is not provided, the default name format used is: `/emr-on-eks-logs/emr-workload/<NAMESPACE>`"
   type        = string