From 0754e3325f20cd46a334fb1b6a46ef013ba09410 Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Tue, 18 Feb 2025 15:58:39 -0800 Subject: [PATCH] add toggle to adjust failurePolicy of pod webhook and documentations --- config/webhook/manifests.yaml | 2 +- docs/deploy/pod_readiness_gate.md | 15 +++++++++++++++ .../templates/webhook.yaml | 2 +- helm/aws-load-balancer-controller/test.yaml | 4 ++++ helm/aws-load-balancer-controller/values.yaml | 5 +++++ 5 files changed, 26 insertions(+), 2 deletions(-) diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 0b1a24bfed..00793b4707 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -11,7 +11,7 @@ webhooks: name: webhook-service namespace: system path: /mutate-v1-pod - failurePolicy: Fail + failurePolicy: Ignore name: mpod.elbv2.k8s.aws rules: - apiGroups: diff --git a/docs/deploy/pod_readiness_gate.md b/docs/deploy/pod_readiness_gate.md index 60ef17a62c..c376895f5c 100644 --- a/docs/deploy/pod_readiness_gate.md +++ b/docs/deploy/pod_readiness_gate.md @@ -47,6 +47,21 @@ The readiness gates have the prefix `target-health.elbv2.k8s.aws` and the contro !!!tip "create ingress or service before pod" To ensure all of your pods in a namespace get the readiness gate config, you need create your Ingress or Service and label the namespace before creating the pods +## FailurePolicy +The `failurePolicy` of a webhook determines how errors, such as unrecognized or timeout errors, are handled by the admission webhook. + +* `failurePolicy: Fail`: When applied to a pod mutation webhook, this setting will prevent the launch of any pods in labeled namespaces if the AWSLoadBalancerController pods are unavailable. While this can help avoid incomplete or faulty deployments, it could also delay the cluster's recovery in extreme scenarios, such as an API Server outage. +* `failurePolicy: Ignore`: Setting this policy allows Kubernetes to proceed with pod deployments even if the AWSLoadBalancerController pods are unavailable. This can lead to availability risks for applications since Kubernetes may terminate application pods before the new pods have become healthy in the TargetGroups + +To strike a balance between reliability and availability, the default failurePolicy for pod mutation webhooks that inject readiness gates is configured as follows: + +* `failurePolicy: Ignore` (for versions > v2.11.0) +* `failurePolicy: Fail` (for versions <= v2.11.0) +You can customize the behavior using Helm chart settings, e.g. `--set podMutatorWebhookConfig.failurePolicy=Fail` + +!!!note "Recommended settings" + For optimal reliability & availability, it is recommended to use `failurePolicy: Fail` combined with an explicit [Object Selector](#object-selector) + ## Object Selector The default webhook configuration matches all pods in the namespaces containing the label `elbv2.k8s.aws/pod-readiness-gate-inject=enabled`. You can modify the webhook configuration further to select specific pods from the labeled namespace by specifying the `objectSelector`. For example, in order to select resources with `elbv2.k8s.aws/pod-readiness-gate-inject: enabled` label, diff --git a/helm/aws-load-balancer-controller/templates/webhook.yaml b/helm/aws-load-balancer-controller/templates/webhook.yaml index 5d27a80830..7d5fcae0fd 100644 --- a/helm/aws-load-balancer-controller/templates/webhook.yaml +++ b/helm/aws-load-balancer-controller/templates/webhook.yaml @@ -19,7 +19,7 @@ webhooks: name: {{ template "aws-load-balancer-controller.webhookService" . }} namespace: {{ $.Release.Namespace }} path: /mutate-v1-pod - failurePolicy: Ignore + failurePolicy: {{ .Values.podMutatorWebhookConfig.failurePolicy }} name: mpod.elbv2.k8s.aws admissionReviewVersions: - v1beta1 diff --git a/helm/aws-load-balancer-controller/test.yaml b/helm/aws-load-balancer-controller/test.yaml index f5074c857e..266ebd14d1 100644 --- a/helm/aws-load-balancer-controller/test.yaml +++ b/helm/aws-load-balancer-controller/test.yaml @@ -353,3 +353,7 @@ serviceMutatorWebhookConfig: operations: - CREATE # - UPDATE + +podMutatorWebhookConfig: + # whether or not to fail the pod creation if the webhook fails + failurePolicy: Ignore \ No newline at end of file diff --git a/helm/aws-load-balancer-controller/values.yaml b/helm/aws-load-balancer-controller/values.yaml index eb28380b27..f3a60f4d3d 100644 --- a/helm/aws-load-balancer-controller/values.yaml +++ b/helm/aws-load-balancer-controller/values.yaml @@ -430,6 +430,11 @@ serviceMutatorWebhookConfig: - CREATE # - UPDATE +# podMutatorWebhookConfig contains configurations specific to the service mutator webhook +podMutatorWebhookConfig: + # whether or not to fail the pod creation if the webhook fails + failurePolicy: Ignore + # serviceTargetENISGTags specifies AWS tags, in addition to the cluster tags, for finding the target ENI SG to which to add inbound rules from NLBs. serviceTargetENISGTags: