From 08411f4d5f06e44f11deb4a1cc38722f06947913 Mon Sep 17 00:00:00 2001 From: magreenbaum Date: Sun, 30 Nov 2025 09:27:06 -0700 Subject: [PATCH 1/2] support scaling strategy and utilization performance index --- README.md | 6 ++++-- examples/private-cluster/README.md | 4 ++-- examples/private-cluster/versions.tf | 2 +- examples/public-cluster/README.md | 4 ++-- examples/public-cluster/versions.tf | 2 +- examples/serverless-cluster/README.md | 4 ++-- examples/serverless-cluster/versions.tf | 2 +- examples/studio/README.md | 4 ++-- examples/studio/versions.tf | 2 +- examples/virtual-cluster/README.md | 4 ++-- examples/virtual-cluster/versions.tf | 2 +- main.tf | 3 +++ modules/serverless/README.md | 4 ++-- modules/serverless/versions.tf | 2 +- modules/studio/README.md | 4 ++-- modules/studio/versions.tf | 2 +- modules/virtual-cluster/README.md | 4 ++-- modules/virtual-cluster/versions.tf | 2 +- variables.tf | 12 ++++++++++++ versions.tf | 2 +- 20 files changed, 44 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 896c748..c118d3a 100644 --- a/README.md +++ b/README.md @@ -332,13 +332,13 @@ Examples codified under the [`examples`](https://github.com/terraform-aws-module | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.19 | +| [aws](#requirement\_aws) | >= 6.22 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.19 | +| [aws](#provider\_aws) | >= 6.22 | ## Modules @@ -435,6 +435,7 @@ No modules. | [release\_label](#input\_release\_label) | Release label for the Amazon EMR release | `string` | `null` | no | | [release\_label\_filters](#input\_release\_label\_filters) | Map of release label filters use to lookup a release label | 
map(object({
    application = optional(string)
    prefix      = optional(string)
  }))
| 
{
  "default": {
    "prefix": "emr-7"
  }
}
| no | | [scale\_down\_behavior](#input\_scale\_down\_behavior) | Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an instance group is resized | `string` | `"TERMINATE_AT_TASK_COMPLETION"` | no | +| [scaling\_strategy](#input\_scaling\_strategy) | Specifies the scaling strategy. When set to ADVANCED, the utilization\_performance\_index argument can be used to configure an advanced scaling strategy. An advanced scaling strategy requires Amazon EMR on EC2 version 7.0 or later. Valid values: ADVANCED, DEFAULT | `string` | `null` | no | | [security\_configuration](#input\_security\_configuration) | Security configuration to create, or attach if `create_security_configuration` is `false`. Only valid for EMR clusters with `release_label` 4.8.0 or greater | `string` | `null` | no | | [security\_configuration\_name](#input\_security\_configuration\_name) | Name of the security configuration to create, or attach if `create_security_configuration` is `false`. Only valid for EMR clusters with `release_label` 4.8.0 or greater | `string` | `null` | no | | [security\_configuration\_use\_name\_prefix](#input\_security\_configuration\_use\_name\_prefix) | Determines whether `security_configuration_name` is used as a prefix | `bool` | `true` | no | @@ -457,6 +458,7 @@ No modules. | [task\_instance\_group](#input\_task\_instance\_group) | Configuration block to use an [Instance Group](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-instance-group-configuration.html#emr-plan-instance-groups) for the [task node type](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-master-core-task-nodes.html#emr-plan-master) | 
object({
    autoscaling_policy  = optional(string)
    bid_price           = optional(string)
    configurations_json = optional(string)
    ebs_config = optional(list(object({
      iops                 = optional(number)
      size                 = optional(number, 256)
      type                 = optional(string, "gp3")
      volumes_per_instance = optional(number)
    })))
    ebs_optimized  = optional(bool, true)
    instance_count = optional(number)
    instance_type  = string
    name           = optional(string)
  })
| `null` | no | | [termination\_protection](#input\_termination\_protection) | Switch on/off termination protection (default is `false`, except when using multiple master nodes). Before attempting to destroy the resource when termination protection is enabled, this configuration must be applied with its value set to `false` | `bool` | `null` | no | | [unhealthy\_node\_replacement](#input\_unhealthy\_node\_replacement) | Whether whether Amazon EMR should gracefully replace core nodes that have degraded within the cluster. Default value is `true` | `bool` | `true` | no | +| [utilization\_performance\_index](#input\_utilization\_performance\_index) | Integer value that represents the advanced scaling strategy. Higher values optimize for performance, while lower values optimize for resource conservation. A value of 50 provides a balance between performance and resource conservation. Required when scaling\_strategy is set to ADVANCED. Valid values: 1, 25, 50, 75, 100 | `number` | `null` | no | | [visible\_to\_all\_users](#input\_visible\_to\_all\_users) | Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default value is `true` | `bool` | `null` | no | | [vpc\_id](#input\_vpc\_id) | The ID of the Amazon Virtual Private Cloud (Amazon VPC) where the security groups will be created | `string` | `""` | no | diff --git a/examples/private-cluster/README.md b/examples/private-cluster/README.md index 4178ca4..803db0e 100644 --- a/examples/private-cluster/README.md +++ b/examples/private-cluster/README.md @@ -29,13 +29,13 @@ Note that this example may create resources which will incur monetary charges on | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.19 | +| [aws](#requirement\_aws) | >= 6.22 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.19 | +| [aws](#provider\_aws) | >= 6.22 | ## Modules diff --git a/examples/private-cluster/versions.tf b/examples/private-cluster/versions.tf index 0d5ee58..0cf1cd7 100644 --- a/examples/private-cluster/versions.tf +++ b/examples/private-cluster/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.19" + version = ">= 6.22" } } } diff --git a/examples/public-cluster/README.md b/examples/public-cluster/README.md index c3ffaec..3834394 100644 --- a/examples/public-cluster/README.md +++ b/examples/public-cluster/README.md @@ -27,13 +27,13 @@ Note that this example may create resources which will incur monetary charges on | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.19 | +| [aws](#requirement\_aws) | >= 6.22 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.19 | +| [aws](#provider\_aws) | >= 6.22 | ## Modules diff --git a/examples/public-cluster/versions.tf b/examples/public-cluster/versions.tf index 0d5ee58..0cf1cd7 100644 --- a/examples/public-cluster/versions.tf +++ b/examples/public-cluster/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.19" + version = ">= 6.22" } } } diff --git a/examples/serverless-cluster/README.md b/examples/serverless-cluster/README.md index 1ae7bdb..f5338b2 100644 --- a/examples/serverless-cluster/README.md +++ b/examples/serverless-cluster/README.md @@ -27,13 +27,13 @@ Note that this example may create resources which will incur monetary charges on | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.19 | +| [aws](#requirement\_aws) | >= 6.22 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.19 | +| [aws](#provider\_aws) | >= 6.22 | ## Modules diff --git a/examples/serverless-cluster/versions.tf b/examples/serverless-cluster/versions.tf index 0d5ee58..0cf1cd7 100644 --- a/examples/serverless-cluster/versions.tf +++ b/examples/serverless-cluster/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.19" + version = ">= 6.22" } } } diff --git a/examples/studio/README.md b/examples/studio/README.md index 833d7f0..2bf8af6 100644 --- a/examples/studio/README.md +++ b/examples/studio/README.md @@ -22,13 +22,13 @@ terraform apply | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.19 | +| [aws](#requirement\_aws) | >= 6.22 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.19 | +| [aws](#provider\_aws) | >= 6.22 | ## Modules diff --git a/examples/studio/versions.tf b/examples/studio/versions.tf index 0d5ee58..0cf1cd7 100644 --- a/examples/studio/versions.tf +++ b/examples/studio/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.19" + version = ">= 6.22" } } } diff --git a/examples/virtual-cluster/README.md b/examples/virtual-cluster/README.md index 89e6801..e04c29f 100644 --- a/examples/virtual-cluster/README.md +++ b/examples/virtual-cluster/README.md @@ -41,7 +41,7 @@ aws emr-containers list-virtual-clusters --region us-west-2 --states ARRESTED \ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.19 | +| [aws](#requirement\_aws) | >= 6.22 | | [kubernetes](#requirement\_kubernetes) | >= 2.38 | | [null](#requirement\_null) | >= 3.0 | | [time](#requirement\_time) | >= 0.7 | @@ -50,7 +50,7 @@ aws emr-containers list-virtual-clusters --region us-west-2 --states ARRESTED \ | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.19 | +| [aws](#provider\_aws) | >= 6.22 | | [null](#provider\_null) | >= 3.0 | ## Modules diff --git a/examples/virtual-cluster/versions.tf b/examples/virtual-cluster/versions.tf index 89104a5..e9df45c 100644 --- a/examples/virtual-cluster/versions.tf +++ b/examples/virtual-cluster/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.19" + version = ">= 6.22" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/main.tf b/main.tf index 848679d..a6a35bc 100644 --- a/main.tf +++ b/main.tf @@ -483,6 +483,9 @@ resource "aws_emr_managed_scaling_policy" "this" { minimum_capacity_units = var.managed_scaling_policy.minimum_capacity_units unit_type = var.managed_scaling_policy.unit_type } + + scaling_strategy = var.scaling_strategy + utilization_performance_index = var.utilization_performance_index } ################################################################################ diff --git a/modules/serverless/README.md b/modules/serverless/README.md index d3eb32e..bb1ec39 100644 --- a/modules/serverless/README.md +++ b/modules/serverless/README.md @@ -132,13 +132,13 @@ Examples codified under the [`examples`](https://github.com/terraform-aws-module | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.19 | +| [aws](#requirement\_aws) | >= 6.22 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.19 | +| [aws](#provider\_aws) | >= 6.22 | ## Modules diff --git a/modules/serverless/versions.tf b/modules/serverless/versions.tf index 0d5ee58..0cf1cd7 100644 --- a/modules/serverless/versions.tf +++ b/modules/serverless/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.19" + version = ">= 6.22" } } } diff --git a/modules/studio/README.md b/modules/studio/README.md index f7daed1..40111c6 100644 --- a/modules/studio/README.md +++ b/modules/studio/README.md @@ -68,13 +68,13 @@ Examples codified under the [`examples`](https://github.com/terraform-aws-module | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.19 | +| [aws](#requirement\_aws) | >= 6.22 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.19 | +| [aws](#provider\_aws) | >= 6.22 | ## Modules diff --git a/modules/studio/versions.tf b/modules/studio/versions.tf index 0d5ee58..0cf1cd7 100644 --- a/modules/studio/versions.tf +++ b/modules/studio/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.19" + version = ">= 6.22" } } } diff --git a/modules/virtual-cluster/README.md b/modules/virtual-cluster/README.md index 72a0fb9..432f67f 100644 --- a/modules/virtual-cluster/README.md +++ b/modules/virtual-cluster/README.md @@ -73,14 +73,14 @@ Examples codified under the [`examples`](https://github.com/terraform-aws-module | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.5.7 | -| [aws](#requirement\_aws) | >= 6.19 | +| [aws](#requirement\_aws) | >= 6.22 | | [kubernetes](#requirement\_kubernetes) | >= 2.38 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 6.19 | +| [aws](#provider\_aws) | >= 6.22 | | [kubernetes](#provider\_kubernetes) | >= 2.38 | ## Modules diff --git a/modules/virtual-cluster/versions.tf b/modules/virtual-cluster/versions.tf index fd1ccc8..e97acdd 100644 --- a/modules/virtual-cluster/versions.tf +++ b/modules/virtual-cluster/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.19" + version = ">= 6.22" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/variables.tf b/variables.tf index 6e13851..f54b9ca 100644 --- a/variables.tf +++ b/variables.tf @@ -406,6 +406,18 @@ variable "managed_scaling_policy" { default = null } +variable "scaling_strategy" { + description = "Specifies the scaling strategy. When set to ADVANCED, the utilization_performance_index argument can be used to configure an advanced scaling strategy. An advanced scaling strategy requires Amazon EMR on EC2 version 7.0 or later. Valid values: ADVANCED, DEFAULT" + type = string + default = null +} + +variable "utilization_performance_index" { + description = "Integer value that represents the advanced scaling strategy. Higher values optimize for performance, while lower values optimize for resource conservation. A value of 50 provides a balance between performance and resource conservation. Required when scaling_strategy is set to ADVANCED. Valid values: 1, 25, 50, 75, 100" + type = number + default = null +} + ################################################################################ # Security Configuration ################################################################################ diff --git a/versions.tf b/versions.tf index 0d5ee58..0cf1cd7 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 6.19" + version = ">= 6.22" } } } From e2a739dbd0521d8a6f6cab0e333adcc7d0f07cff Mon Sep 17 00:00:00 2001 From: magreenbaum Date: Sun, 30 Nov 2025 14:25:26 -0700 Subject: [PATCH 2/2] feedback changes --- README.md | 4 +--- main.tf | 4 ++-- variables.tf | 14 ++------------ 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index c118d3a..54bc1de 100644 --- a/README.md +++ b/README.md @@ -419,7 +419,7 @@ No modules. | [list\_steps\_states](#input\_list\_steps\_states) | List of [step states](https://docs.aws.amazon.com/emr/latest/APIReference/API_StepStatus.html) used to filter returned steps | `list(string)` | `[]` | no | | [log\_encryption\_kms\_key\_id](#input\_log\_encryption\_kms\_key\_id) | AWS KMS customer master key (CMK) key ID or arn used for encrypting log files. This attribute is only available with EMR version 5.30.0 and later, excluding EMR 6.0.0 | `string` | `null` | no | | [log\_uri](#input\_log\_uri) | S3 bucket to write the log files of the job flow. If a value is not provided, logs are not created | `string` | `null` | no | -| [managed\_scaling\_policy](#input\_managed\_scaling\_policy) | Compute limit configuration for a [Managed Scaling Policy](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-managed-scaling.html) | 
object({
    maximum_capacity_units          = number
    maximum_core_capacity_units     = optional(number)
    maximum_ondemand_capacity_units = optional(number)
    minimum_capacity_units          = number
    unit_type                       = string
  })
| `null` | no | +| [managed\_scaling\_policy](#input\_managed\_scaling\_policy) | Compute limit configuration for a [Managed Scaling Policy](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-managed-scaling.html) | 
object({
    maximum_capacity_units          = number
    maximum_core_capacity_units     = optional(number)
    maximum_ondemand_capacity_units = optional(number)
    minimum_capacity_units          = number
    unit_type                       = string
    scaling_strategy                = optional(string)
    utilization_performance_index   = optional(number)
  })
| `null` | no | | [managed\_security\_group\_name](#input\_managed\_security\_group\_name) | Name to use on manged security group created. Note - `-master`, `-slave`, and `-service` will be appended to this name to distinguish | `string` | `null` | no | | [managed\_security\_group\_tags](#input\_managed\_security\_group\_tags) | A map of additional tags to add to the security group created | `map(string)` | `{}` | no | | [managed\_security\_group\_use\_name\_prefix](#input\_managed\_security\_group\_use\_name\_prefix) | Determines whether the security group name (`security_group_name`) is used as a prefix | `bool` | `true` | no | @@ -435,7 +435,6 @@ No modules. | [release\_label](#input\_release\_label) | Release label for the Amazon EMR release | `string` | `null` | no | | [release\_label\_filters](#input\_release\_label\_filters) | Map of release label filters use to lookup a release label | 
map(object({
    application = optional(string)
    prefix      = optional(string)
  }))
| 
{
  "default": {
    "prefix": "emr-7"
  }
}
| no | | [scale\_down\_behavior](#input\_scale\_down\_behavior) | Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an instance group is resized | `string` | `"TERMINATE_AT_TASK_COMPLETION"` | no | -| [scaling\_strategy](#input\_scaling\_strategy) | Specifies the scaling strategy. When set to ADVANCED, the utilization\_performance\_index argument can be used to configure an advanced scaling strategy. An advanced scaling strategy requires Amazon EMR on EC2 version 7.0 or later. Valid values: ADVANCED, DEFAULT | `string` | `null` | no | | [security\_configuration](#input\_security\_configuration) | Security configuration to create, or attach if `create_security_configuration` is `false`. Only valid for EMR clusters with `release_label` 4.8.0 or greater | `string` | `null` | no | | [security\_configuration\_name](#input\_security\_configuration\_name) | Name of the security configuration to create, or attach if `create_security_configuration` is `false`. Only valid for EMR clusters with `release_label` 4.8.0 or greater | `string` | `null` | no | | [security\_configuration\_use\_name\_prefix](#input\_security\_configuration\_use\_name\_prefix) | Determines whether `security_configuration_name` is used as a prefix | `bool` | `true` | no | @@ -458,7 +457,6 @@ No modules. | [task\_instance\_group](#input\_task\_instance\_group) | Configuration block to use an [Instance Group](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-instance-group-configuration.html#emr-plan-instance-groups) for the [task node type](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-master-core-task-nodes.html#emr-plan-master) | 
object({
    autoscaling_policy  = optional(string)
    bid_price           = optional(string)
    configurations_json = optional(string)
    ebs_config = optional(list(object({
      iops                 = optional(number)
      size                 = optional(number, 256)
      type                 = optional(string, "gp3")
      volumes_per_instance = optional(number)
    })))
    ebs_optimized  = optional(bool, true)
    instance_count = optional(number)
    instance_type  = string
    name           = optional(string)
  })
| `null` | no | | [termination\_protection](#input\_termination\_protection) | Switch on/off termination protection (default is `false`, except when using multiple master nodes). Before attempting to destroy the resource when termination protection is enabled, this configuration must be applied with its value set to `false` | `bool` | `null` | no | | [unhealthy\_node\_replacement](#input\_unhealthy\_node\_replacement) | Whether whether Amazon EMR should gracefully replace core nodes that have degraded within the cluster. Default value is `true` | `bool` | `true` | no | -| [utilization\_performance\_index](#input\_utilization\_performance\_index) | Integer value that represents the advanced scaling strategy. Higher values optimize for performance, while lower values optimize for resource conservation. A value of 50 provides a balance between performance and resource conservation. Required when scaling\_strategy is set to ADVANCED. Valid values: 1, 25, 50, 75, 100 | `number` | `null` | no | | [visible\_to\_all\_users](#input\_visible\_to\_all\_users) | Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default value is `true` | `bool` | `null` | no | | [vpc\_id](#input\_vpc\_id) | The ID of the Amazon Virtual Private Cloud (Amazon VPC) where the security groups will be created | `string` | `""` | no | diff --git a/main.tf b/main.tf index a6a35bc..8a4f5ec 100644 --- a/main.tf +++ b/main.tf @@ -484,8 +484,8 @@ resource "aws_emr_managed_scaling_policy" "this" { unit_type = var.managed_scaling_policy.unit_type } - scaling_strategy = var.scaling_strategy - utilization_performance_index = var.utilization_performance_index + scaling_strategy = var.managed_scaling_policy.scaling_strategy + utilization_performance_index = var.managed_scaling_policy.utilization_performance_index } ################################################################################ diff --git a/variables.tf b/variables.tf index f54b9ca..8cecb20 100644 --- a/variables.tf +++ b/variables.tf @@ -402,22 +402,12 @@ variable "managed_scaling_policy" { maximum_ondemand_capacity_units = optional(number) minimum_capacity_units = number unit_type = string + scaling_strategy = optional(string) + utilization_performance_index = optional(number) }) default = null } -variable "scaling_strategy" { - description = "Specifies the scaling strategy. When set to ADVANCED, the utilization_performance_index argument can be used to configure an advanced scaling strategy. An advanced scaling strategy requires Amazon EMR on EC2 version 7.0 or later. Valid values: ADVANCED, DEFAULT" - type = string - default = null -} - -variable "utilization_performance_index" { - description = "Integer value that represents the advanced scaling strategy. Higher values optimize for performance, while lower values optimize for resource conservation. A value of 50 provides a balance between performance and resource conservation. Required when scaling_strategy is set to ADVANCED. Valid values: 1, 25, 50, 75, 100" - type = number - default = null -} - ################################################################################ # Security Configuration ################################################################################