add toggle to adjust failurePolicy of pod webhook and documentations (kubernetes-sigs#4063)

M00nF1sh · zac-nixon · commit 3b250808aa76 · 2025-04-28T15:34:58.000-07:00
the vulcheck failed due to other reasons.
diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml
@@ -4,128 +4,128 @@ kind: MutatingWebhookConfiguration
 metadata:
   name: webhook
 webhooks:
-- admissionReviewVersions:
-  - v1beta1
-  clientConfig:
-    service:
-      name: webhook-service
-      namespace: system
-      path: /mutate-v1-pod
-  failurePolicy: Fail
-  name: mpod.elbv2.k8s.aws
-  rules:
-  - apiGroups:
-    - ""
-    apiVersions:
-    - v1
-    operations:
-    - CREATE
-    resources:
-    - pods
-  sideEffects: None
-- admissionReviewVersions:
-  - v1beta1
-  clientConfig:
-    service:
-      name: webhook-service
-      namespace: system
-      path: /mutate-v1-service
-  failurePolicy: Fail
-  name: mservice.elbv2.k8s.aws
-  rules:
-  - apiGroups:
-    - ""
-    apiVersions:
-    - v1
-    operations:
-    - CREATE
-    resources:
-    - services
-  sideEffects: None
-- admissionReviewVersions:
-  - v1beta1
-  clientConfig:
-    service:
-      name: webhook-service
-      namespace: system
-      path: /mutate-elbv2-k8s-aws-v1beta1-targetgroupbinding
-  failurePolicy: Fail
-  name: mtargetgroupbinding.elbv2.k8s.aws
-  rules:
-  - apiGroups:
-    - elbv2.k8s.aws
-    apiVersions:
-    - v1beta1
-    operations:
-    - CREATE
-    - UPDATE
-    resources:
-    - targetgroupbindings
-  sideEffects: None
+  - admissionReviewVersions:
+      - v1beta1
+    clientConfig:
+      service:
+        name: webhook-service
+        namespace: system
+        path: /mutate-v1-pod
+    failurePolicy: Ignore
+    name: mpod.elbv2.k8s.aws
+    rules:
+      - apiGroups:
+          - ""
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+        resources:
+          - pods
+    sideEffects: None
+  - admissionReviewVersions:
+      - v1beta1
+    clientConfig:
+      service:
+        name: webhook-service
+        namespace: system
+        path: /mutate-v1-service
+    failurePolicy: Fail
+    name: mservice.elbv2.k8s.aws
+    rules:
+      - apiGroups:
+          - ""
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+        resources:
+          - services
+    sideEffects: None
+  - admissionReviewVersions:
+      - v1beta1
+    clientConfig:
+      service:
+        name: webhook-service
+        namespace: system
+        path: /mutate-elbv2-k8s-aws-v1beta1-targetgroupbinding
+    failurePolicy: Fail
+    name: mtargetgroupbinding.elbv2.k8s.aws
+    rules:
+      - apiGroups:
+          - elbv2.k8s.aws
+        apiVersions:
+          - v1beta1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - targetgroupbindings
+    sideEffects: None
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
-  name: validating-webhook-configuration
+  name: webhook
 webhooks:
-- admissionReviewVersions:
-  - v1beta1
-  clientConfig:
-    service:
-      name: webhook-service
-      namespace: system
-      path: /validate-elbv2-k8s-aws-v1beta1-ingressclassparams
-  failurePolicy: Fail
-  name: vingressclassparams.elbv2.k8s.aws
-  rules:
-  - apiGroups:
-    - elbv2.k8s.aws
-    apiVersions:
-    - v1beta1
-    operations:
-    - CREATE
-    - UPDATE
-    resources:
-    - ingressclassparams
-  sideEffects: None
-- admissionReviewVersions:
-  - v1beta1
-  clientConfig:
-    service:
-      name: webhook-service
-      namespace: system
-      path: /validate-elbv2-k8s-aws-v1beta1-targetgroupbinding
-  failurePolicy: Fail
-  name: vtargetgroupbinding.elbv2.k8s.aws
-  rules:
-  - apiGroups:
-    - elbv2.k8s.aws
-    apiVersions:
-    - v1beta1
-    operations:
-    - CREATE
-    - UPDATE
-    resources:
-    - targetgroupbindings
-  sideEffects: None
-- admissionReviewVersions:
-  - v1beta1
-  clientConfig:
-    service:
-      name: webhook-service
-      namespace: system
-      path: /validate-networking-v1-ingress
-  failurePolicy: Fail
-  matchPolicy: Equivalent
-  name: vingress.elbv2.k8s.aws
-  rules:
-  - apiGroups:
-    - networking.k8s.io
-    apiVersions:
-    - v1
-    operations:
-    - CREATE
-    - UPDATE
-    resources:
-    - ingresses
-  sideEffects: None
+  - admissionReviewVersions:
+      - v1beta1
+    clientConfig:
+      service:
+        name: webhook-service
+        namespace: system
+        path: /validate-elbv2-k8s-aws-v1beta1-ingressclassparams
+    failurePolicy: Fail
+    name: vingressclassparams.elbv2.k8s.aws
+    rules:
+      - apiGroups:
+          - elbv2.k8s.aws
+        apiVersions:
+          - v1beta1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - ingressclassparams
+    sideEffects: None
+  - admissionReviewVersions:
+      - v1beta1
+    clientConfig:
+      service:
+        name: webhook-service
+        namespace: system
+        path: /validate-elbv2-k8s-aws-v1beta1-targetgroupbinding
+    failurePolicy: Fail
+    name: vtargetgroupbinding.elbv2.k8s.aws
+    rules:
+      - apiGroups:
+          - elbv2.k8s.aws
+        apiVersions:
+          - v1beta1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - targetgroupbindings
+    sideEffects: None
+  - admissionReviewVersions:
+      - v1beta1
+    clientConfig:
+      service:
+        name: webhook-service
+        namespace: system
+        path: /validate-networking-v1-ingress
+    failurePolicy: Fail
+    matchPolicy: Equivalent
+    name: vingress.elbv2.k8s.aws
+    rules:
+      - apiGroups:
+          - networking.k8s.io
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - ingresses
+    sideEffects: None
diff --git a/docs/deploy/pod_readiness_gate.md b/docs/deploy/pod_readiness_gate.md
@@ -47,6 +47,21 @@ The readiness gates have the prefix `target-health.elbv2.k8s.aws` and the contro
 !!!tip "create ingress or service before pod"
     To ensure all of your pods in a namespace get the readiness gate config, you need create your Ingress or Service and label the namespace before creating the pods
 
+## FailurePolicy
+The `failurePolicy` of a webhook determines how errors, such as unrecognized or timeout errors, are handled by the admission webhook.
+
+* `failurePolicy: Fail`: When applied to a pod mutation webhook, this setting will prevent the launch of any pods in labeled namespaces if the AWSLoadBalancerController pods are unavailable. While this can help avoid incomplete or faulty deployments, it could also delay the cluster's recovery in extreme scenarios, such as an API Server outage.
+* `failurePolicy: Ignore`: Setting this policy allows Kubernetes to proceed with pod deployments even if the AWSLoadBalancerController pods are unavailable. This can lead to availability risks for applications since Kubernetes may terminate application pods before the new pods have become healthy in the TargetGroups
+
+To strike a balance between reliability and availability, the default failurePolicy for pod mutation webhooks that inject readiness gates is configured as follows:
+
+* `failurePolicy: Ignore` (for versions > v2.11.0)
+* `failurePolicy: Fail` (for versions <= v2.11.0)
+You can customize the behavior using Helm chart settings, e.g. `--set podMutatorWebhookConfig.failurePolicy=Fail`
+
+!!!note "Recommended settings"
+    For optimal reliability & availability, it is recommended to use `failurePolicy: Fail` combined with an explicit [Object Selector](#object-selector)
+
 ## Object Selector
 The default webhook configuration matches all pods in the namespaces containing the label `elbv2.k8s.aws/pod-readiness-gate-inject=enabled`. You can modify the webhook configuration further
 to select specific pods from the labeled namespace by specifying the `objectSelector`. For example, in order to select resources with `elbv2.k8s.aws/pod-readiness-gate-inject: enabled` label,
diff --git a/helm/aws-load-balancer-controller/templates/webhook.yaml b/helm/aws-load-balancer-controller/templates/webhook.yaml
@@ -19,7 +19,7 @@ webhooks:
       name: {{ template "aws-load-balancer-controller.webhookService" . }}
       namespace: {{ $.Release.Namespace }}
       path: /mutate-v1-pod
-  failurePolicy: Ignore
+  failurePolicy: {{ .Values.podMutatorWebhookConfig.failurePolicy }}
   name: mpod.elbv2.k8s.aws
   admissionReviewVersions:
   - v1beta1
diff --git a/helm/aws-load-balancer-controller/test.yaml b/helm/aws-load-balancer-controller/test.yaml
@@ -353,3 +353,7 @@ serviceMutatorWebhookConfig:
   operations:
   - CREATE
   # - UPDATE
+
+podMutatorWebhookConfig:
+  # whether or not to fail the pod creation if the webhook fails
+  failurePolicy: Ignore
diff --git a/helm/aws-load-balancer-controller/values.yaml b/helm/aws-load-balancer-controller/values.yaml
@@ -430,6 +430,11 @@ serviceMutatorWebhookConfig:
   - CREATE
     # - UPDATE
 
+# podMutatorWebhookConfig contains configurations specific to the service mutator webhook
+podMutatorWebhookConfig:
+  # whether or not to fail the pod creation if the webhook fails
+  failurePolicy: Ignore
+
 # serviceTargetENISGTags specifies AWS tags, in addition to the cluster tags, for finding the target ENI SG to which to add inbound rules from NLBs.
 serviceTargetENISGTags:
 
