Skip to content

Commit 5a7af88

Browse files
wweiwei-liwweiwei-li
committed
Add prometheus metrics
1 parent 526b830 commit 5a7af88

28 files changed

+693
-188
lines changed

‎controllers/elbv2/targetgroupbinding_controller.go‎

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,11 @@ package controllers
1919
import (
2020
"context"
2121
"fmt"
22+
"time"
23+
2224
discv1 "k8s.io/api/discovery/v1"
2325
"sigs.k8s.io/controller-runtime/pkg/handler"
2426
"sigs.k8s.io/controller-runtime/pkg/reconcile"
25-
"time"
2627

2728
"github.com/aws/aws-sdk-go-v2/aws"
2829
"github.com/pkg/errors"
@@ -31,12 +32,14 @@ import (
3132
"k8s.io/client-go/util/workqueue"
3233
"sigs.k8s.io/aws-load-balancer-controller/controllers/elbv2/eventhandlers"
3334
"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
35+
errmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/error"
3436
"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
3537
"sigs.k8s.io/aws-load-balancer-controller/pkg/runtime"
3638
"sigs.k8s.io/aws-load-balancer-controller/pkg/targetgroupbinding"
3739
"sigs.k8s.io/controller-runtime/pkg/controller"
3840

3941
"github.com/go-logr/logr"
42+
lbcmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/lbc"
4043
ctrl "sigs.k8s.io/controller-runtime"
4144
"sigs.k8s.io/controller-runtime/pkg/client"
4245

@@ -51,7 +54,7 @@ const (
5154
// NewTargetGroupBindingReconciler constructs new targetGroupBindingReconciler
5255
func NewTargetGroupBindingReconciler(k8sClient client.Client, eventRecorder record.EventRecorder, finalizerManager k8s.FinalizerManager,
5356
tgbResourceManager targetgroupbinding.ResourceManager, config config.ControllerConfig, deferredTargetGroupBindingReconciler DeferredTargetGroupBindingReconciler,
54-
logger logr.Logger) *targetGroupBindingReconciler {
57+
logger logr.Logger, metricsCollector lbcmetrics.MetricCollector) *targetGroupBindingReconciler {
5558

5659
return &targetGroupBindingReconciler{
5760
k8sClient: k8sClient,
@@ -60,6 +63,7 @@ func NewTargetGroupBindingReconciler(k8sClient client.Client, eventRecorder reco
6063
tgbResourceManager: tgbResourceManager,
6164
deferredTargetGroupBindingReconciler: deferredTargetGroupBindingReconciler,
6265
logger: logger,
66+
metricsCollector: metricsCollector,
6367

6468
maxConcurrentReconciles: config.TargetGroupBindingMaxConcurrentReconciles,
6569
maxExponentialBackoffDelay: config.TargetGroupBindingMaxExponentialBackoffDelay,
@@ -75,6 +79,7 @@ type targetGroupBindingReconciler struct {
7579
tgbResourceManager targetgroupbinding.ResourceManager
7680
deferredTargetGroupBindingReconciler DeferredTargetGroupBindingReconciler
7781
logger logr.Logger
82+
metricsCollector lbcmetrics.MetricCollector
7883

7984
maxConcurrentReconciles int
8085
maxExponentialBackoffDelay time.Duration
@@ -110,25 +115,36 @@ func (r *targetGroupBindingReconciler) reconcile(ctx context.Context, req reconc
110115
}
111116

112117
func (r *targetGroupBindingReconciler) reconcileTargetGroupBinding(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error {
113-
if err := r.finalizerManager.AddFinalizers(ctx, tgb, targetGroupBindingFinalizer); err != nil {
118+
var err error
119+
finalizerFn := func() {
120+
err = r.finalizerManager.AddFinalizers(ctx, tgb, targetGroupBindingFinalizer)
121+
}
122+
r.metricsCollector.ObserveControllerReconcileLatency("targetGroupBinding", "add_finalizers", finalizerFn)
123+
if err != nil {
114124
r.eventRecorder.Event(tgb, corev1.EventTypeWarning, k8s.TargetGroupBindingEventReasonFailedAddFinalizer, fmt.Sprintf("Failed add finalizer due to %v", err))
115-
return err
125+
return errmetrics.NewErrorWithMetrics("targetGroupBinding", "add_finalizers_error", err, r.metricsCollector)
116126
}
117127

118-
deferred, err := r.tgbResourceManager.Reconcile(ctx, tgb)
119-
128+
var deferred bool
129+
tgbResourceFn := func() {
130+
deferred, err = r.tgbResourceManager.Reconcile(ctx, tgb)
131+
}
132+
r.metricsCollector.ObserveControllerReconcileLatency("targetGroupBinding", "reconcile_targetgroupblinding", tgbResourceFn)
120133
if err != nil {
121-
return err
134+
return errmetrics.NewErrorWithMetrics("targetGroupBinding", "reconcile_targetgroupblinding_error", err, r.metricsCollector)
122135
}
123136

124137
if deferred {
125138
r.deferredTargetGroupBindingReconciler.Enqueue(tgb)
126139
return nil
127140
}
128141

129-
if err := r.updateTargetGroupBindingStatus(ctx, tgb); err != nil {
130-
r.eventRecorder.Event(tgb, corev1.EventTypeWarning, k8s.TargetGroupBindingEventReasonFailedUpdateStatus, fmt.Sprintf("Failed update status due to %v", err))
131-
return err
142+
updateTargetGroupBindingStatusFn := func() {
143+
err = r.updateTargetGroupBindingStatus(ctx, tgb)
144+
}
145+
defer r.metricsCollector.ObserveControllerReconcileLatency("targetGroupBinding", "update_status", updateTargetGroupBindingStatusFn)
146+
if err != nil {
147+
return errmetrics.NewErrorWithMetrics("targetGroupBinding", "update_status_error", err, r.metricsCollector)
132148
}
133149

134150
r.eventRecorder.Event(tgb, corev1.EventTypeNormal, k8s.TargetGroupBindingEventReasonSuccessfullyReconciled, "Successfully reconciled")

‎controllers/ingress/group_controller.go‎

Lines changed: 59 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package ingress
33
import (
44
"context"
55
"fmt"
6+
67
"sigs.k8s.io/controller-runtime/pkg/reconcile"
78

89
"github.com/go-logr/logr"
@@ -21,8 +22,10 @@ import (
2122
"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy"
2223
elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2"
2324
"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking"
25+
errmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/error"
2426
"sigs.k8s.io/aws-load-balancer-controller/pkg/ingress"
2527
"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
28+
lbcmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/lbc"
2629
"sigs.k8s.io/aws-load-balancer-controller/pkg/model/core"
2730
elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2"
2831
networkingpkg "sigs.k8s.io/aws-load-balancer-controller/pkg/networking"
@@ -48,7 +51,7 @@ func NewGroupReconciler(cloud services.Cloud, k8sClient client.Client, eventReco
4851
finalizerManager k8s.FinalizerManager, networkingSGManager networkingpkg.SecurityGroupManager,
4952
networkingSGReconciler networkingpkg.SecurityGroupReconciler, subnetsResolver networkingpkg.SubnetsResolver,
5053
elbv2TaggingManager elbv2deploy.TaggingManager, controllerConfig config.ControllerConfig, backendSGProvider networkingpkg.BackendSGProvider,
51-
sgResolver networkingpkg.SecurityGroupResolver, logger logr.Logger) *groupReconciler {
54+
sgResolver networkingpkg.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector) *groupReconciler {
5255

5356
annotationParser := annotations.NewSuffixAnnotationParser(annotations.AnnotationPrefixIngress)
5457
authConfigBuilder := ingress.NewDefaultAuthConfigBuilder(annotationParser)
@@ -61,10 +64,10 @@ func NewGroupReconciler(cloud services.Cloud, k8sClient client.Client, eventReco
6164
authConfigBuilder, enhancedBackendBuilder, trackingProvider, elbv2TaggingManager, controllerConfig.FeatureGates,
6265
cloud.VpcID(), controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags,
6366
controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, backendSGProvider, sgResolver,
64-
controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, controllerConfig.IngressConfig.AllowedCertificateAuthorityARNs, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), logger)
67+
controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, controllerConfig.IngressConfig.AllowedCertificateAuthorityARNs, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), logger, metricsCollector)
6568
stackMarshaller := deploy.NewDefaultStackMarshaller()
6669
stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager,
67-
controllerConfig, ingressTagPrefix, logger)
70+
controllerConfig, ingressTagPrefix, logger, metricsCollector, controllerName)
6871
classLoader := ingress.NewDefaultClassLoader(k8sClient, true)
6972
classAnnotationMatcher := ingress.NewDefaultClassAnnotationMatcher(controllerConfig.IngressConfig.IngressClass)
7073
manageIngressesWithoutIngressClass := controllerConfig.IngressConfig.IngressClass == ""
@@ -83,6 +86,8 @@ func NewGroupReconciler(cloud services.Cloud, k8sClient client.Client, eventReco
8386
groupLoader: groupLoader,
8487
groupFinalizerManager: groupFinalizerManager,
8588
logger: logger,
89+
metricsCollector: metricsCollector,
90+
controllerName: controllerName,
8691

8792
maxConcurrentReconciles: controllerConfig.IngressConfig.MaxConcurrentReconciles,
8893
}
@@ -102,6 +107,8 @@ type groupReconciler struct {
102107
groupLoader ingress.GroupLoader
103108
groupFinalizerManager ingress.FinalizerManager
104109
logger logr.Logger
110+
metricsCollector lbcmetrics.MetricCollector
111+
controllerName string
105112

106113
maxConcurrentReconciles int
107114
}
@@ -121,33 +128,53 @@ func (r *groupReconciler) Reconcile(ctx context.Context, req reconcile.Request)
121128

122129
func (r *groupReconciler) reconcile(ctx context.Context, req reconcile.Request) error {
123130
ingGroupID := ingress.DecodeGroupIDFromReconcileRequest(req)
124-
ingGroup, err := r.groupLoader.Load(ctx, ingGroupID)
131+
var err error
132+
var ingGroup ingress.Group
133+
loadIngressFn := func() {
134+
ingGroup, err = r.groupLoader.Load(ctx, ingGroupID)
135+
}
136+
r.metricsCollector.ObserveControllerReconcileLatency("ingress", "fetch_ingress", loadIngressFn)
125137
if err != nil {
126-
return err
138+
return errmetrics.NewErrorWithMetrics("ingress", "fetch_ingress_error", err, r.metricsCollector)
127139
}
128140

129-
if err := r.groupFinalizerManager.AddGroupFinalizer(ctx, ingGroupID, ingGroup.Members); err != nil {
141+
addFinalizerFn := func() {
142+
err = r.groupFinalizerManager.AddGroupFinalizer(ctx, ingGroupID, ingGroup.Members)
143+
}
144+
r.metricsCollector.ObserveControllerReconcileLatency("ingress", "add_group_finalizer", addFinalizerFn)
145+
if err != nil {
130146
r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedAddFinalizer, fmt.Sprintf("Failed add finalizer due to %v", err))
131-
return err
147+
return errmetrics.NewErrorWithMetrics("ingress", "add_group_finalizer_error", err, r.metricsCollector)
148+
}
149+
150+
var lb *elbv2model.LoadBalancer
151+
buildAndDeployModelFn := func() {
152+
_, lb, err = r.buildAndDeployModel(ctx, ingGroup)
132153
}
133-
_, lb, err := r.buildAndDeployModel(ctx, ingGroup)
154+
r.metricsCollector.ObserveControllerReconcileLatency("ingress", "build_and_deploy_model", buildAndDeployModelFn)
134155
if err != nil {
135-
return err
156+
return errmetrics.NewErrorWithMetrics("ingress", "build_and_deploy_model_error", err, r.metricsCollector)
136157
}
137158

138159
if len(ingGroup.Members) > 0 && lb != nil {
139-
lbDNS, err := lb.DNSName().Resolve(ctx)
140-
if err != nil {
141-
return err
142-
}
143-
if err := r.updateIngressGroupStatus(ctx, ingGroup, lbDNS); err != nil {
144-
r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedUpdateStatus, fmt.Sprintf("Failed update status due to %v", err))
145-
return err
160+
dnsResolveAndUpdateStatus := func() {
161+
lbDNS, err := lb.DNSName().Resolve(ctx)
162+
if err != nil {
163+
return
164+
}
165+
if err := r.updateIngressGroupStatus(ctx, ingGroup, lbDNS); err != nil {
166+
r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedUpdateStatus, fmt.Sprintf("Failed update status due to %v", err))
167+
}
146168
}
169+
r.metricsCollector.ObserveControllerReconcileLatency("ingress", "dns_resolve_and_update_status", dnsResolveAndUpdateStatus)
147170
}
148171

149172
if len(ingGroup.InactiveMembers) > 0 {
150-
if err := r.groupFinalizerManager.RemoveGroupFinalizer(ctx, ingGroupID, ingGroup.InactiveMembers); err != nil {
173+
removeGroupFinalizerFn := func() {
174+
err = r.groupFinalizerManager.RemoveGroupFinalizer(ctx, ingGroupID, ingGroup.InactiveMembers)
175+
}
176+
r.metricsCollector.ObserveControllerReconcileLatency("ingress", "remove_group_finalizer", removeGroupFinalizerFn)
177+
if err != nil {
151178
r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedRemoveFinalizer, fmt.Sprintf("Failed remove finalizer due to %v", err))
152179
return err
153180
}
@@ -158,7 +185,15 @@ func (r *groupReconciler) reconcile(ctx context.Context, req reconcile.Request)
158185
}
159186

160187
func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingress.Group) (core.Stack, *elbv2model.LoadBalancer, error) {
161-
stack, lb, secrets, backendSGRequired, err := r.modelBuilder.Build(ctx, ingGroup)
188+
var stack core.Stack
189+
var lb *elbv2model.LoadBalancer
190+
var secrets []types.NamespacedName
191+
var backendSGRequired bool
192+
var err error
193+
buildModelFn := func() {
194+
stack, lb, secrets, backendSGRequired, err = r.modelBuilder.Build(ctx, ingGroup, r.metricsCollector)
195+
}
196+
r.metricsCollector.ObserveControllerReconcileLatency("ingress", "build_model", buildModelFn)
162197
if err != nil {
163198
r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err))
164199
return nil, nil, err
@@ -170,7 +205,11 @@ func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingr
170205
}
171206
r.logger.Info("successfully built model", "model", stackJSON)
172207

173-
if err := r.stackDeployer.Deploy(ctx, stack); err != nil {
208+
deployModelFn := func() {
209+
err = r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "ingress")
210+
}
211+
r.metricsCollector.ObserveControllerReconcileLatency("ingress", "deploy_model", deployModelFn)
212+
if err != nil {
174213
var requeueNeededAfter *runtime.RequeueNeededAfter
175214
if errors.As(err, &requeueNeededAfter) {
176215
return nil, nil, err
@@ -186,7 +225,7 @@ func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingr
186225
inactiveResources = append(inactiveResources, k8s.ToSliceOfNamespacedNames(ingGroup.Members)...)
187226
}
188227
if err := r.backendSGProvider.Release(ctx, networkingpkg.ResourceTypeIngress, inactiveResources); err != nil {
189-
return nil, nil, err
228+
return nil, nil, errmetrics.NewErrorWithMetrics("ingress", "release_auto_generated_backend_sg_error", err, r.metricsCollector)
190229
}
191230
return stack, lb, nil
192231
}

0 commit comments

Comments
 (0)