Update rule management to avoid sporadic 503 errors

Author: shraddhabang

.go-version

@@ -1 +1 @@
-1.22.8
+1.23.0

docs/install/iam_policy.json

@@ -42,7 +42,8 @@
                 "elasticloadbalancing:DescribeTags",
                 "elasticloadbalancing:DescribeTrustStores",
                 "elasticloadbalancing:DescribeListenerAttributes",
-                "elasticloadbalancing:DescribeCapacityReservation"
+                "elasticloadbalancing:DescribeCapacityReservation",
+                "elasticloadbalancing:SetRulePriorities"
             ],
             "Resource": "*"
         },

docs/install/iam_policy_cn.json

@@ -42,7 +42,8 @@
                 "elasticloadbalancing:DescribeTags",
                 "elasticloadbalancing:DescribeTrustStores",
                 "elasticloadbalancing:DescribeListenerAttributes",
-                "elasticloadbalancing:DescribeCapacityReservation"
+                "elasticloadbalancing:DescribeCapacityReservation",
+                "elasticloadbalancing:SetRulePriorities"
             ],
             "Resource": "*"
         },

docs/install/iam_policy_iso.json

@@ -39,7 +39,8 @@
                 "elasticloadbalancing:DescribeTargetGroups",
                 "elasticloadbalancing:DescribeTargetGroupAttributes",
                 "elasticloadbalancing:DescribeTargetHealth",
-                "elasticloadbalancing:DescribeTags"
+                "elasticloadbalancing:DescribeTags",
+                "elasticloadbalancing:SetRulePriorities"
             ],
             "Resource": "*"
         },

docs/install/iam_policy_isob.json

@@ -39,7 +39,8 @@
                 "elasticloadbalancing:DescribeTargetGroups",
                 "elasticloadbalancing:DescribeTargetGroupAttributes",
                 "elasticloadbalancing:DescribeTargetHealth",
-                "elasticloadbalancing:DescribeTags"
+                "elasticloadbalancing:DescribeTags",
+                "elasticloadbalancing:SetRulePriorities"
             ],
             "Resource": "*"
         },

go.mod

@@ -1,6 +1,6 @@
 module sigs.k8s.io/aws-load-balancer-controller
 
-go 1.22.8
+go 1.23.0
 
 require (
 	github.com/aws/aws-sdk-go v1.55.5

pkg/aws/services/elbv2.go

@@ -50,7 +50,8 @@ type ELBV2 interface {
 	DescribeRulesWithContext(ctx context.Context, input *elasticloadbalancingv2.DescribeRulesInput) (*elasticloadbalancingv2.DescribeRulesOutput, error)
 	CreateRuleWithContext(ctx context.Context, input *elasticloadbalancingv2.CreateRuleInput) (*elasticloadbalancingv2.CreateRuleOutput, error)
 	DeleteRuleWithContext(ctx context.Context, input *elasticloadbalancingv2.DeleteRuleInput) (*elasticloadbalancingv2.DeleteRuleOutput, error)
-	ModifyRuleWithContext(ctx context.Context, inout *elasticloadbalancingv2.ModifyRuleInput) (*elasticloadbalancingv2.ModifyRuleOutput, error)
+	ModifyRuleWithContext(ctx context.Context, input *elasticloadbalancingv2.ModifyRuleInput) (*elasticloadbalancingv2.ModifyRuleOutput, error)
+	SetRulePrioritiesWithContext(ctx context.Context, input *elasticloadbalancingv2.SetRulePrioritiesInput) (*elasticloadbalancingv2.SetRulePrioritiesOutput, error)
 	RegisterTargetsWithContext(ctx context.Context, input *elasticloadbalancingv2.RegisterTargetsInput) (*elasticloadbalancingv2.RegisterTargetsOutput, error)
 	DeregisterTargetsWithContext(ctx context.Context, input *elasticloadbalancingv2.DeregisterTargetsInput) (*elasticloadbalancingv2.DeregisterTargetsOutput, error)
 	DescribeTrustStoresWithContext(ctx context.Context, input *elasticloadbalancingv2.DescribeTrustStoresInput) (*elasticloadbalancingv2.DescribeTrustStoresOutput, error)
@@ -163,6 +164,14 @@ func (c *elbv2Client) DeleteRuleWithContext(ctx context.Context, input *elasticl
 	return client.DeleteRule(ctx, input)
 }
 
+func (c *elbv2Client) SetRulePrioritiesWithContext(ctx context.Context, input *elasticloadbalancingv2.SetRulePrioritiesInput) (*elasticloadbalancingv2.SetRulePrioritiesOutput, error) {
+	client, err := c.awsClientsProvider.GetELBv2Client(ctx, "SetRulePriorities")
+	if err != nil {
+		return nil, err
+	}
+	return client.SetRulePriorities(ctx, input)
+}
+
 func (c *elbv2Client) CreateRuleWithContext(ctx context.Context, input *elasticloadbalancingv2.CreateRuleInput) (*elasticloadbalancingv2.CreateRuleOutput, error) {
 	client, err := c.getClient(ctx, "CreateRule")
 	if err != nil {

pkg/aws/services/elbv2_mocks.go

@@ -6,24 +6,26 @@ import (
 	elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2"
 	elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types"
 	"github.com/go-logr/logr"
-	"github.com/google/go-cmp/cmp"
 	"github.com/pkg/errors"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking"
-	elbv2equality "sigs.k8s.io/aws-load-balancer-controller/pkg/equality/elbv2"
 	elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/runtime"
+	"slices"
+	"strconv"
 	"time"
 )
 
 // ListenerRuleManager is responsible for create/update/delete ListenerRule resources.
 type ListenerRuleManager interface {
-	Create(ctx context.Context, resLR *elbv2model.ListenerRule) (elbv2model.ListenerRuleStatus, error)
+	Create(ctx context.Context, resLR *elbv2model.ListenerRule, desiredActionsAndConditions *resLRDesiredActionsAndConditionsPair) (elbv2model.ListenerRuleStatus, error)
 
 	Update(ctx context.Context, resLR *elbv2model.ListenerRule, sdkLR ListenerRuleWithTags) (elbv2model.ListenerRuleStatus, error)
 
 	Delete(ctx context.Context, sdkLR ListenerRuleWithTags) error
+
+	SetRulePriorities(ctx context.Context, sdkLR []ListenerRuleWithTags, lastAvailablePriority int32) (int32, error)
 }
 
 // NewDefaultListenerRuleManager constructs new defaultListenerRuleManager.
@@ -54,8 +56,8 @@ type defaultListenerRuleManager struct {
 	waitLSExistenceTimeout      time.Duration
 }
 
-func (m *defaultListenerRuleManager) Create(ctx context.Context, resLR *elbv2model.ListenerRule) (elbv2model.ListenerRuleStatus, error) {
-	req, err := buildSDKCreateListenerRuleInput(resLR.Spec, m.featureGates)
+func (m *defaultListenerRuleManager) Create(ctx context.Context, resLR *elbv2model.ListenerRule, desiredActionsAndConditions *resLRDesiredActionsAndConditionsPair) (elbv2model.ListenerRuleStatus, error) {
+	req, err := buildSDKCreateListenerRuleInput(resLR.Spec, desiredActionsAndConditions, m.featureGates)
 	if err != nil {
 		return elbv2model.ListenerRuleStatus{}, err
 	}
@@ -96,9 +98,6 @@ func (m *defaultListenerRuleManager) Update(ctx context.Context, resLR *elbv2mod
 			return elbv2model.ListenerRuleStatus{}, err
 		}
 	}
-	if err := m.updateSDKListenerRuleWithSettings(ctx, resLR, sdkLR); err != nil {
-		return elbv2model.ListenerRuleStatus{}, err
-	}
 	return buildResListenerRuleStatus(sdkLR), nil
 }
 
@@ -116,30 +115,21 @@ func (m *defaultListenerRuleManager) Delete(ctx context.Context, sdkLR ListenerR
 	return nil
 }
 
-func (m *defaultListenerRuleManager) updateSDKListenerRuleWithSettings(ctx context.Context, resLR *elbv2model.ListenerRule, sdkLR ListenerRuleWithTags) error {
-	desiredActions, err := buildSDKActions(resLR.Spec.Actions, m.featureGates)
-	if err != nil {
-		return err
-	}
-	desiredConditions := buildSDKRuleConditions(resLR.Spec.Conditions)
-	if !isSDKListenerRuleSettingsDrifted(resLR.Spec, sdkLR, desiredActions, desiredConditions) {
-		return nil
-	}
-
-	req := buildSDKModifyListenerRuleInput(resLR.Spec, desiredActions, desiredConditions)
-	req.RuleArn = sdkLR.ListenerRule.RuleArn
-	m.logger.Info("modifying listener rule",
-		"stackID", resLR.Stack().StackID(),
-		"resourceID", resLR.ID(),
-		"arn", awssdk.ToString(sdkLR.ListenerRule.RuleArn))
-	if _, err := m.elbv2Client.ModifyRuleWithContext(ctx, req); err != nil {
-		return err
-	}
-	m.logger.Info("modified listener rule",
-		"stackID", resLR.Stack().StackID(),
-		"resourceID", resLR.ID(),
-		"arn", awssdk.ToString(sdkLR.ListenerRule.RuleArn))
-	return nil
+func (m *defaultListenerRuleManager) SetRulePriorities(ctx context.Context, unmatchedSDKLRs []ListenerRuleWithTags, lastAvailablePriority int32) (int32, error) {
+	for _, sdkLR := range slices.Backward(unmatchedSDKLRs) {
+		//Update rule priorities
+		sdkLR.ListenerRule.Priority = awssdk.String(strconv.Itoa(int(lastAvailablePriority)))
+		lastAvailablePriority--
+	}
+	req := buildSDKSetRulePrioritiesInput(unmatchedSDKLRs)
+	m.logger.Info("setting listener rule priorities",
+		"rule priority pairs", req.RulePriorities)
+	if _, err := m.elbv2Client.SetRulePrioritiesWithContext(ctx, req); err != nil {
+		return lastAvailablePriority, err
+	}
+	m.logger.Info("setting listener rule priorities complete",
+		"rule priority pairs", req.RulePriorities)
+	return lastAvailablePriority, nil
 }
 
 func (m *defaultListenerRuleManager) updateSDKListenerRuleWithTags(ctx context.Context, resLR *elbv2model.ListenerRule, sdkLR ListenerRuleWithTags) error {
@@ -149,20 +139,7 @@ func (m *defaultListenerRuleManager) updateSDKListenerRuleWithTags(ctx context.C
 		WithIgnoredTagKeys(m.externalManagedTags))
 }
 
-func isSDKListenerRuleSettingsDrifted(lrSpec elbv2model.ListenerRuleSpec, sdkLR ListenerRuleWithTags,
-	desiredActions []elbv2types.Action, desiredConditions []elbv2types.RuleCondition) bool {
-
-	if !cmp.Equal(desiredActions, sdkLR.ListenerRule.Actions, elbv2equality.CompareOptionForActions()) {
-		return true
-	}
-	if !cmp.Equal(desiredConditions, sdkLR.ListenerRule.Conditions, elbv2equality.CompareOptionForRuleConditions()) {
-		return true
-	}
-
-	return false
-}
-
-func buildSDKCreateListenerRuleInput(lrSpec elbv2model.ListenerRuleSpec, featureGates config.FeatureGates) (*elbv2sdk.CreateRuleInput, error) {
+func buildSDKCreateListenerRuleInput(lrSpec elbv2model.ListenerRuleSpec, desiredActionsAndConditions *resLRDesiredActionsAndConditionsPair, featureGates config.FeatureGates) (*elbv2sdk.CreateRuleInput, error) {
 	ctx := context.Background()
 	lsARN, err := lrSpec.ListenerARN.Resolve(ctx)
 	if err != nil {
@@ -171,22 +148,38 @@ func buildSDKCreateListenerRuleInput(lrSpec elbv2model.ListenerRuleSpec, feature
 	sdkObj := &elbv2sdk.CreateRuleInput{}
 	sdkObj.ListenerArn = awssdk.String(lsARN)
 	sdkObj.Priority = awssdk.Int32(lrSpec.Priority)
-	actions, err := buildSDKActions(lrSpec.Actions, featureGates)
-	if err != nil {
-		return nil, err
+	if desiredActionsAndConditions != nil && desiredActionsAndConditions.desiredActions != nil {
+		sdkObj.Actions = desiredActionsAndConditions.desiredActions
+	} else {
+		actions, err := buildSDKActions(lrSpec.Actions, featureGates)
+		if err != nil {
+			return nil, err
+		}
+		sdkObj.Actions = actions
+	}
+	if desiredActionsAndConditions != nil && desiredActionsAndConditions.desiredConditions != nil {
+		sdkObj.Conditions = desiredActionsAndConditions.desiredConditions
+	} else {
+		sdkObj.Conditions = buildSDKRuleConditions(lrSpec.Conditions)
 	}
-	sdkObj.Actions = actions
-	sdkObj.Conditions = buildSDKRuleConditions(lrSpec.Conditions)
 	return sdkObj, nil
 }
 
-func buildSDKModifyListenerRuleInput(_ elbv2model.ListenerRuleSpec, desiredActions []elbv2types.Action, desiredConditions []elbv2types.RuleCondition) *elbv2sdk.ModifyRuleInput {
-	sdkObj := &elbv2sdk.ModifyRuleInput{}
-	sdkObj.Actions = desiredActions
-	sdkObj.Conditions = desiredConditions
+func buildSDKSetRulePrioritiesInput(sdkLRs []ListenerRuleWithTags) *elbv2sdk.SetRulePrioritiesInput {
+	var rulePriorities []elbv2types.RulePriorityPair
+	for _, sdkLR := range sdkLRs {
+		p, _ := strconv.ParseInt(awssdk.ToString(sdkLR.ListenerRule.Priority), 10, 32)
+		rulePriorityPair := elbv2types.RulePriorityPair{
+			RuleArn:  sdkLR.ListenerRule.RuleArn,
+			Priority: awssdk.Int32(int32(p)),
+		}
+		rulePriorities = append(rulePriorities, rulePriorityPair)
+	}
+	sdkObj := &elbv2sdk.SetRulePrioritiesInput{
+		RulePriorities: rulePriorities,
+	}
 	return sdkObj
 }
-
 func buildResListenerRuleStatus(sdkLR ListenerRuleWithTags) elbv2model.ListenerRuleStatus {
 	return elbv2model.ListenerRuleStatus{
 		RuleARN: awssdk.ToString(sdkLR.ListenerRule.RuleArn),