Add support for capacity unit reservation for load balancers

Author: shraddhabang

.gitignore

@@ -24,3 +24,4 @@ site
 *.swo
 *~
 *.bak
+scripts/aws_sdk_model_override/*

apis/elbv2/v1beta1/ingressclassparams_types.go

@@ -101,6 +101,12 @@ type Listener struct {
 	ListenerAttributes []Attribute `json:"listenerAttributes,omitempty"`
 }
 
+// Information about a load balancer capacity reservation.
+type MinimumLoadBalancerCapacity struct {
+	// The Capacity Units Value.
+	CapacityUnits int32 `json:"capacityUnits"`
+}
+
 // IngressClassParamsSpec defines the desired state of IngressClassParams
 type IngressClassParamsSpec struct {
 	// CertificateArn specifies the ARN of the certificates for all Ingresses that belong to IngressClass with this IngressClassParams.
@@ -146,6 +152,10 @@ type IngressClassParamsSpec struct {
 	// Listeners define a list of listeners with their protocol, port and attributes.
 	// +optional
 	Listeners []Listener `json:"listeners,omitempty"`
+
+	// MinimumLoadBalancerCapacity define the capacity reservation for LoadBalancers for all Ingress that belong to IngressClass with this IngressClassParams.
+	// +optional
+	MinimumLoadBalancerCapacity *MinimumLoadBalancerCapacity `json:"minimumLoadBalancerCapacity,omitempty"`
 }
 
 // +kubebuilder:object:root=true

apis/elbv2/v1beta1/zz_generated.deepcopy.go

@@ -133,6 +133,18 @@ spec:
                   - value
                   type: object
                 type: array
+              minimumLoadBalancerCapacity:
+                description: MinimumLoadBalancerCapacity define the capacity reservation
+                  for LoadBalancers for all Ingress that belong to IngressClass with
+                  this IngressClassParams.
+                properties:
+                  capacityUnits:
+                    description: The Capacity Units Value.
+                    format: int32
+                    type: integer
+                required:
+                - capacityUnits
+                type: object
               namespaceSelector:
                 description: |-
                   NamespaceSelector restrict the namespaces of Ingresses that are allowed to specify the IngressClass with this IngressClassParams.

config/crd/bases/elbv2.k8s.aws_ingressclassparams.yaml

@@ -170,6 +170,10 @@ func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingr
 	r.logger.Info("successfully built model", "model", stackJSON)
 
 	if err := r.stackDeployer.Deploy(ctx, stack); err != nil {
+		var requeueNeededAfter *runtime.RequeueNeededAfter
+		if errors.As(err, &requeueNeededAfter) {
+			return nil, nil, err
+		}
 		r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedDeployModel, fmt.Sprintf("Failed deploy model due to %v", err))
 		return nil, nil, err
 	}

controllers/ingress/group_controller.go

@@ -124,6 +124,10 @@ func (r *serviceReconciler) buildModel(ctx context.Context, svc *corev1.Service)
 
 func (r *serviceReconciler) deployModel(ctx context.Context, svc *corev1.Service, stack core.Stack) error {
 	if err := r.stackDeployer.Deploy(ctx, stack); err != nil {
+		var requeueNeededAfter *runtime.RequeueNeededAfter
+		if errors.As(err, &requeueNeededAfter) {
+			return err
+		}
 		r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedDeployModel, fmt.Sprintf("Failed deploy model due to %v", err))
 		return err
 	}

controllers/service/service_controller.go

@@ -104,6 +104,7 @@ Currently, you can set only 1 namespace to watch in this flag. See [this Kuberne
 | [sync-period](#sync-period)                                                     | duration                        | 10h0m0s                                    | Period at which the controller forces the repopulation of its local object stores                                                              |
 | targetgroupbinding-max-concurrent-reconciles                                    | int                       | 3                                          | Maximum number of concurrently running reconcile loops for targetGroupBinding                                                                  |
 | targetgroupbinding-max-exponential-backoff-delay                                | duration              | 16m40s                                     | Maximum duration of exponential backoff for targetGroupBinding reconcile failures                                                              |
+| [lb-stabilization-monitor-interval](#lb-stabilization-monitor-interval)         | duration                        | 2m                               | Interval at which the controller monitors the state of load balancer after creation
 | tolerate-non-existent-backend-service                                           | boolean                         | true                                       | Whether to allow rules which refer to backend services that do not exist (When enabled, it will return 503 error if backend service not exist) |
 | tolerate-non-existent-backend-action                                            | boolean                         | true                                       | Whether to allow rules which refer to backend actions that do not exist (When enabled, it will return 503 error if backend action not exist)   |
 | watch-namespace                                                                 | string                          |                                            | Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched.                                      |
@@ -137,6 +138,9 @@ Once disabled:
 
 As best practice, we do not recommend users to manually modify the resources managed by the controller. And users should not depend on the controller auto-reconciliation to revert the manual modification, or to mitigate any security risks.
 
+### lb-stabilization-monitor-interval
+`--lb-stabilization-monitor-interval` defines a fixed interval for the controller to monitor the state of load balancer after the creation for stabilization, default to 2m. It monitors the load balancer state so that once it becomes active it can make the required updates like capacity reservation for the active load balancer. It calls DescribeLoadBalancer API at a fixed interval to monitor the state. Please be mindful that lower value will result into frequent calls which may incur unnecessary AWS API usage.
+
 ### waf-addons
 By default, the controller assumes sole ownership of the WAF addons associated to the provisioned ALBs, via the flag `--enable-waf` and `--enable-wafv2`.
 And the users should disable them accordingly if they want a third party like AWS Firewall Manager to associate or remove the WAF-ACL of the ALBs.
@@ -177,3 +181,4 @@ There are a set of key=value pairs that describe AWS load balancer controller fe
 | NLBHealthCheckAdvancedConfiguration   | string                          | true          | Enable or disable advanced health check configuration for NLB, for example health check timeout                                                                                      |
 | ALBSingleSubnet                       | string                          | false         | If enabled, controller will allow using only 1 subnet for provisioning ALB, which need to get whitelisted by ELB in advance                                                          |
 | NLBSecurityGroup                      | string                          | true          | Enable or disable all NLB security groups actions including frontend sg creation, backend sg creation, and backend sg modifications                                                  |
+| LBCapacityReservation                 | string                          | true          | Enable or disable the capacity reservation feature on ALB and NLB

docs/deploy/configurations.md

@@ -60,6 +60,7 @@ You can add annotations to kubernetes Ingress and Service objects to customize t
 | [alb.ingress.kubernetes.io/target-node-labels](#target-node-labels)                                   | stringMap                   |N/A| Ingress,Service | N/A           |
 | [alb.ingress.kubernetes.io/mutual-authentication](#mutual-authentication)                             | json                        |N/A| Ingress         | Exclusive     |
 | [alb.ingress.kubernetes.io/multi-cluster-target-group](#multi-cluster-target-group)                   | boolean                     |N/A| Ingress, Service | N/A           |
+| [alb.ingress.kubernetes.io/minimum-load-balancer-capacity](#load-balancer-capacity-reservation)                       | stringMap                   |N/A| Ingress         | Exclusive     |
 
 ## IngressGroup
 IngressGroup feature enables you to group multiple Ingress resources together.
@@ -922,6 +923,26 @@ In addition, you can use annotations to specify additional tags
         alb.ingress.kubernetes.io/tags: Environment=dev,Team=test
         ```
 
+## Capacity Unit Reservation
+Load balancer capacity unit reservation can be configured via following annotations:
+
+- <a name="load-balancer-capacity-reservation">`alb.ingress.kubernetes.io/minimum-load-balancer-capacity`</a> specifies the
+  [Capacity Unit Reservation](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/capacity-unit-reservation.html) to be configured.
+
+    !!!example
+        - set the capacity unit reservation to 1000
+          ```
+          alb.ingress.kubernetes.io/minimum-load-balancer-capacity: CapacityUnits=1000
+          ```
+        - reset the capacity unit reservation
+          ```
+          alb.ingress.kubernetes.io/minimum-load-balancer-capacity: CapacityUnits=0
+          ```
+
+    !!!note "Notes"
+        - If you specify this annotation, but remove it later, the capacity unit reservation is not reset. You need to reset the capacity by setting the capacity units to zero as show in the example above.
+        - If users do not want the controller to manage the capacity unit reservation on load balancer, they can disable the feature by setting controller command line feature gate flag ```--feature-gates=LBCapacityReservation=true```
+
 ## Addons
 
 - <a name="waf-acl-id">`alb.ingress.kubernetes.io/waf-acl-id`</a> specifies the identifier for the Amazon WAF Classic web ACL.

docs/guide/ingress/annotations.md

@@ -140,6 +140,16 @@ You can use IngressClassParams to enforce settings for a set of Ingresses.
     spec:
       certificateArn: ['arn:aws:acm:us-east-1:123456789:certificate/test-arn-1','arn:aws:acm:us-east-1:123456789:certificate/test-arn-2']
     ```
+    - with minimumLoadBalancerCapacity.capacityUnits
+    ```
+    apiVersion: elbv2.k8s.aws/v1beta1
+    kind: IngressClassParams
+    metadata:
+      name: class2048-config
+    spec:
+      minimumLoadBalancerCapacity:
+        capacityUnits: 1000
+    ```
 
 ### IngressClassParams specification
 
@@ -233,3 +243,12 @@ Cluster administrators can use `loadBalancerAttributes` field to specify the [Lo
 
 1. If `loadBalancerAttributes` is set, the attributes defined will be applied to the load balancer that belong to this IngressClass. If you specify invalid keys or values for the load balancer attributes, the controller will fail to reconcile ingresses belonging to the particular ingress class.
 2. If `loadBalancerAttributes` un-specified, Ingresses with this IngressClass can continue to use `alb.ingress.kubernetes.io/load-balancer-attributes` annotation to specify the load balancer attributes.
+
+#### spec.minimumLoadBalancerCapacity
+
+Cluster administrators can use the optional `minimumLoadBalancerCapacity` field to specify the capacity reservation for the load balancers that belong to this IngressClass.
+They may specify `capacityUnits`. If the field is specified, LBC will ignore the `alb.ingress.kubernetes.io/minimum-load-balancer-capacity annotation` annotation.
+
+##### spec.minimumLoadBalancerCapacity.capacityUnits
+
+If `capacityUnits` is specified, it must be to valid positive value greater than 0. If set to 0, the LBC will reset the capacity reservation for the load balancer.

docs/guide/ingress/ingress_class.md

@@ -56,6 +56,7 @@
 | [service.beta.kubernetes.io/aws-load-balancer-multi-cluster-target-group](#multi-cluster-target-group)             | boolean                | false                    | If specified, the controller will only operate on targets that exist within the cluster, ignoring targets from other sources.                                                                                                       |
 | [service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat](#enable-prefix-for-ipv6-source-nat)         | string                  | off                       | Optional annotation. dualstack lb only. Allowed values - on and off                                                                                                                                                                 |
 | [service.beta.kubernetes.io/aws-load-balancer-source-nat-ipv6-prefixes](#source-nat-ipv6-prefixes)        | stringList                  |                           | Optional annotation. dualstack lb only. This annotation is only applicable when user has to set the service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat to "on". Length must match the number of subnets |
+| [service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity](#load-balancer-capacity-reservation)                 | stringMap                  |                           |
 
 ## Traffic Routing
 Traffic Routing can be controlled with following annotations:
@@ -579,6 +580,25 @@ Load balancer access can be controlled via following annotations:
         service.beta.kubernetes.io/aws-load-balancer-inbound-sg-rules-on-private-link-traffic: "off"
         ```
 
+## Capacity Unit Reservation
+Load balancer capacity unit reservation can be configured via following annotations:
+
+- <a name="load-balancer-capacity-reservation">`service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity`</a> specifies the
+  [Capacity Unit Reservation](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/capacity-unit-reservation.html) to be configured.
+
+    !!!example
+        - set the capacity unit reservation to 1000
+          ```
+          service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity: CapacityUnits=3000
+          ```
+        - reset the capacity unit reservation
+          ```
+          service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity: CapacityUnits=0
+          ```
+    
+    !!!note "Notes"
+         - If you specify this annotation, but remove it later, the capacity unit reservation is not reset. You need to reset the capacity by setting the capacity units to zero as show in the example above.
+         - If users do not want the controller to manage the capacity unit reservation on load balancer, they can disable the feature by setting controller command line feature gate flag ```--feature-gates=LBCapacityReservation=true```
 
 ## Legacy Cloud Provider
 The AWS Load Balancer Controller manages Kubernetes Services in a compatible way with the AWS cloud provider's legacy service controller.