Skip to content

Commit 30d0554

Browse files
wweiwei-liwweiwei-li
committed
add prometheusmetrics
1 parent 57472c4 commit 30d0554

27 files changed

+597
-128
lines changed

‎controllers/elbv2/targetgroupbinding_controller.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,11 @@ package controllers
1919
import (
2020
"context"
2121
"fmt"
22+
"time"
23+
2224
discv1 "k8s.io/api/discovery/v1"
2325
"sigs.k8s.io/controller-runtime/pkg/handler"
2426
"sigs.k8s.io/controller-runtime/pkg/reconcile"
25-
"time"
2627

2728
"github.com/aws/aws-sdk-go-v2/aws"
2829
"github.com/pkg/errors"
@@ -37,6 +38,7 @@ import (
3738
"sigs.k8s.io/controller-runtime/pkg/controller"
3839

3940
"github.com/go-logr/logr"
41+
lbcmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/lbc"
4042
ctrl "sigs.k8s.io/controller-runtime"
4143
"sigs.k8s.io/controller-runtime/pkg/client"
4244

@@ -51,7 +53,7 @@ const (
5153
// NewTargetGroupBindingReconciler constructs new targetGroupBindingReconciler
5254
func NewTargetGroupBindingReconciler(k8sClient client.Client, eventRecorder record.EventRecorder, finalizerManager k8s.FinalizerManager,
5355
tgbResourceManager targetgroupbinding.ResourceManager, config config.ControllerConfig, deferredTargetGroupBindingReconciler DeferredTargetGroupBindingReconciler,
54-
logger logr.Logger) *targetGroupBindingReconciler {
56+
logger logr.Logger, metricsCollector lbcmetrics.MetricCollector) *targetGroupBindingReconciler {
5557

5658
return &targetGroupBindingReconciler{
5759
k8sClient: k8sClient,
@@ -60,6 +62,7 @@ func NewTargetGroupBindingReconciler(k8sClient client.Client, eventRecorder reco
6062
tgbResourceManager: tgbResourceManager,
6163
deferredTargetGroupBindingReconciler: deferredTargetGroupBindingReconciler,
6264
logger: logger,
65+
metricsCollector: metricsCollector,
6366

6467
maxConcurrentReconciles: config.TargetGroupBindingMaxConcurrentReconciles,
6568
maxExponentialBackoffDelay: config.TargetGroupBindingMaxExponentialBackoffDelay,
@@ -75,6 +78,7 @@ type targetGroupBindingReconciler struct {
7578
tgbResourceManager targetgroupbinding.ResourceManager
7679
deferredTargetGroupBindingReconciler DeferredTargetGroupBindingReconciler
7780
logger logr.Logger
81+
metricsCollector lbcmetrics.MetricCollector
7882

7983
maxConcurrentReconciles int
8084
maxExponentialBackoffDelay time.Duration
@@ -110,22 +114,26 @@ func (r *targetGroupBindingReconciler) reconcile(ctx context.Context, req reconc
110114
}
111115

112116
func (r *targetGroupBindingReconciler) reconcileTargetGroupBinding(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error {
117+
defer r.metricsCollector.ObserveControllerReconcileLatency("targetGroupBinding", "add finalizers")
113118
if err := r.finalizerManager.AddFinalizers(ctx, tgb, targetGroupBindingFinalizer); err != nil {
114119
r.eventRecorder.Event(tgb, corev1.EventTypeWarning, k8s.TargetGroupBindingEventReasonFailedAddFinalizer, fmt.Sprintf("Failed add finalizer due to %v", err))
115120
return err
116121
}
117122

123+
defer r.metricsCollector.ObserveControllerReconcileLatency("targetGroupBinding", "reconcile")
118124
deferred, err := r.tgbResourceManager.Reconcile(ctx, tgb)
119125

120126
if err != nil {
121127
return err
122128
}
123129

124130
if deferred {
131+
defer r.metricsCollector.ObserveControllerReconcileLatency("targetGroupBinding", "deferred reconcile")
125132
r.deferredTargetGroupBindingReconciler.Enqueue(tgb)
126133
return nil
127134
}
128135

136+
defer r.metricsCollector.ObserveControllerReconcileLatency("targetGroupBinding", "update status")
129137
if err := r.updateTargetGroupBindingStatus(ctx, tgb); err != nil {
130138
r.eventRecorder.Event(tgb, corev1.EventTypeWarning, k8s.TargetGroupBindingEventReasonFailedUpdateStatus, fmt.Sprintf("Failed update status due to %v", err))
131139
return err

‎controllers/ingress/group_controller.go‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package ingress
33
import (
44
"context"
55
"fmt"
6+
67
"sigs.k8s.io/controller-runtime/pkg/reconcile"
78

89
"github.com/go-logr/logr"
@@ -23,6 +24,7 @@ import (
2324
"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking"
2425
"sigs.k8s.io/aws-load-balancer-controller/pkg/ingress"
2526
"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
27+
lbcmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/lbc"
2628
"sigs.k8s.io/aws-load-balancer-controller/pkg/model/core"
2729
elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2"
2830
networkingpkg "sigs.k8s.io/aws-load-balancer-controller/pkg/networking"
@@ -48,7 +50,7 @@ func NewGroupReconciler(cloud services.Cloud, k8sClient client.Client, eventReco
4850
finalizerManager k8s.FinalizerManager, networkingSGManager networkingpkg.SecurityGroupManager,
4951
networkingSGReconciler networkingpkg.SecurityGroupReconciler, subnetsResolver networkingpkg.SubnetsResolver,
5052
elbv2TaggingManager elbv2deploy.TaggingManager, controllerConfig config.ControllerConfig, backendSGProvider networkingpkg.BackendSGProvider,
51-
sgResolver networkingpkg.SecurityGroupResolver, logger logr.Logger) *groupReconciler {
53+
sgResolver networkingpkg.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector) *groupReconciler {
5254

5355
annotationParser := annotations.NewSuffixAnnotationParser(annotations.AnnotationPrefixIngress)
5456
authConfigBuilder := ingress.NewDefaultAuthConfigBuilder(annotationParser)
@@ -61,10 +63,10 @@ func NewGroupReconciler(cloud services.Cloud, k8sClient client.Client, eventReco
6163
authConfigBuilder, enhancedBackendBuilder, trackingProvider, elbv2TaggingManager, controllerConfig.FeatureGates,
6264
cloud.VpcID(), controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags,
6365
controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, backendSGProvider, sgResolver,
64-
controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, controllerConfig.IngressConfig.AllowedCertificateAuthorityARNs, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), logger)
66+
controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, controllerConfig.IngressConfig.AllowedCertificateAuthorityARNs, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), logger, metricsCollector)
6567
stackMarshaller := deploy.NewDefaultStackMarshaller()
6668
stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager,
67-
controllerConfig, ingressTagPrefix, logger)
69+
controllerConfig, ingressTagPrefix, logger, metricsCollector)
6870
classLoader := ingress.NewDefaultClassLoader(k8sClient, true)
6971
classAnnotationMatcher := ingress.NewDefaultClassAnnotationMatcher(controllerConfig.IngressConfig.IngressClass)
7072
manageIngressesWithoutIngressClass := controllerConfig.IngressConfig.IngressClass == ""
@@ -83,6 +85,7 @@ func NewGroupReconciler(cloud services.Cloud, k8sClient client.Client, eventReco
8385
groupLoader: groupLoader,
8486
groupFinalizerManager: groupFinalizerManager,
8587
logger: logger,
88+
metricsCollector: metricsCollector,
8689

8790
maxConcurrentReconciles: controllerConfig.IngressConfig.MaxConcurrentReconciles,
8891
}
@@ -102,6 +105,7 @@ type groupReconciler struct {
102105
groupLoader ingress.GroupLoader
103106
groupFinalizerManager ingress.FinalizerManager
104107
logger logr.Logger
108+
metricsCollector lbcmetrics.MetricCollector
105109

106110
maxConcurrentReconciles int
107111
}
@@ -120,21 +124,26 @@ func (r *groupReconciler) Reconcile(ctx context.Context, req reconcile.Request)
120124
}
121125

122126
func (r *groupReconciler) reconcile(ctx context.Context, req reconcile.Request) error {
127+
defer r.metricsCollector.ObserveControllerReconcileLatency("ingress", "fetch ingress")
123128
ingGroupID := ingress.DecodeGroupIDFromReconcileRequest(req)
124129
ingGroup, err := r.groupLoader.Load(ctx, ingGroupID)
125130
if err != nil {
126131
return err
127132
}
128133

134+
defer r.metricsCollector.ObserveControllerReconcileLatency("ingress", "add group finalizer")
129135
if err := r.groupFinalizerManager.AddGroupFinalizer(ctx, ingGroupID, ingGroup.Members); err != nil {
130136
r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedAddFinalizer, fmt.Sprintf("Failed add finalizer due to %v", err))
131137
return err
132138
}
139+
140+
defer r.metricsCollector.ObserveControllerReconcileLatency("ingress", "build and deploy model")
133141
_, lb, err := r.buildAndDeployModel(ctx, ingGroup)
134142
if err != nil {
135143
return err
136144
}
137145

146+
defer r.metricsCollector.ObserveControllerReconcileLatency("ingress", "DNS resolve and update status")
138147
if len(ingGroup.Members) > 0 && lb != nil {
139148
lbDNS, err := lb.DNSName().Resolve(ctx)
140149
if err != nil {
@@ -146,6 +155,7 @@ func (r *groupReconciler) reconcile(ctx context.Context, req reconcile.Request)
146155
}
147156
}
148157

158+
defer r.metricsCollector.ObserveControllerReconcileLatency("ingress", "remove group finalizer")
149159
if len(ingGroup.InactiveMembers) > 0 {
150160
if err := r.groupFinalizerManager.RemoveGroupFinalizer(ctx, ingGroupID, ingGroup.InactiveMembers); err != nil {
151161
r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedRemoveFinalizer, fmt.Sprintf("Failed remove finalizer due to %v", err))
@@ -158,7 +168,8 @@ func (r *groupReconciler) reconcile(ctx context.Context, req reconcile.Request)
158168
}
159169

160170
func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingress.Group) (core.Stack, *elbv2model.LoadBalancer, error) {
161-
stack, lb, secrets, backendSGRequired, err := r.modelBuilder.Build(ctx, ingGroup)
171+
defer r.metricsCollector.ObserveControllerReconcileLatency("ingress", "build model")
172+
stack, lb, secrets, backendSGRequired, err := r.modelBuilder.Build(ctx, ingGroup, r.metricsCollector)
162173
if err != nil {
163174
r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err))
164175
return nil, nil, err
@@ -169,12 +180,13 @@ func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingr
169180
return nil, nil, err
170181
}
171182
r.logger.Info("successfully built model", "model", stackJSON)
172-
173-
if err := r.stackDeployer.Deploy(ctx, stack); err != nil {
183+
defer r.metricsCollector.ObserveControllerReconcileLatency("ingress", "deploy model")
184+
if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector); err != nil {
174185
var requeueNeededAfter *runtime.RequeueNeededAfter
175186
if errors.As(err, &requeueNeededAfter) {
176187
return nil, nil, err
177188
}
189+
r.metricsCollector.ObserveControllerReconcileError("ingress", "deploy model error", "deploy model error")
178190
r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedDeployModel, fmt.Sprintf("Failed deploy model due to %v", err))
179191
return nil, nil, err
180192
}
@@ -186,6 +198,7 @@ func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingr
186198
inactiveResources = append(inactiveResources, k8s.ToSliceOfNamespacedNames(ingGroup.Members)...)
187199
}
188200
if err := r.backendSGProvider.Release(ctx, networkingpkg.ResourceTypeIngress, inactiveResources); err != nil {
201+
r.metricsCollector.ObserveControllerReconcileError("ingress", "deploy model error", "release auto-generated backend SG error")
189202
return nil, nil, err
190203
}
191204
return stack, lb, nil

‎controllers/service/service_controller.go‎

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package service
33
import (
44
"context"
55
"fmt"
6+
67
"sigs.k8s.io/controller-runtime/pkg/reconcile"
78

89
"github.com/go-logr/logr"
@@ -18,6 +19,7 @@ import (
1819
elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2"
1920
"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking"
2021
"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
22+
lbcmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/lbc"
2123
"sigs.k8s.io/aws-load-balancer-controller/pkg/model/core"
2224
elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2"
2325
"sigs.k8s.io/aws-load-balancer-controller/pkg/networking"
@@ -39,17 +41,17 @@ func NewServiceReconciler(cloud services.Cloud, k8sClient client.Client, eventRe
3941
finalizerManager k8s.FinalizerManager, networkingSGManager networking.SecurityGroupManager,
4042
networkingSGReconciler networking.SecurityGroupReconciler, subnetsResolver networking.SubnetsResolver,
4143
vpcInfoProvider networking.VPCInfoProvider, elbv2TaggingManager elbv2deploy.TaggingManager, controllerConfig config.ControllerConfig,
42-
backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger) *serviceReconciler {
44+
backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector) *serviceReconciler {
4345

4446
annotationParser := annotations.NewSuffixAnnotationParser(serviceAnnotationPrefix)
4547
trackingProvider := tracking.NewDefaultProvider(serviceTagPrefix, controllerConfig.ClusterName)
4648
serviceUtils := service.NewServiceUtils(annotationParser, serviceFinalizer, controllerConfig.ServiceConfig.LoadBalancerClass, controllerConfig.FeatureGates)
4749
modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider,
4850
elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags,
4951
controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils,
50-
backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, logger)
52+
backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, logger, metricsCollector)
5153
stackMarshaller := deploy.NewDefaultStackMarshaller()
52-
stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, serviceTagPrefix, logger)
54+
stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, serviceTagPrefix, logger, metricsCollector)
5355
return &serviceReconciler{
5456
k8sClient: k8sClient,
5557
eventRecorder: eventRecorder,
@@ -65,6 +67,7 @@ func NewServiceReconciler(cloud services.Cloud, k8sClient client.Client, eventRe
6567
logger: logger,
6668

6769
maxConcurrentReconciles: controllerConfig.ServiceMaxConcurrentReconciles,
70+
metricsCollector: metricsCollector,
6871
}
6972
}
7073

@@ -77,10 +80,11 @@ type serviceReconciler struct {
7780
serviceUtils service.ServiceUtils
7881
backendSGProvider networking.BackendSGProvider
7982

80-
modelBuilder service.ModelBuilder
81-
stackMarshaller deploy.StackMarshaller
82-
stackDeployer deploy.StackDeployer
83-
logger logr.Logger
83+
modelBuilder service.ModelBuilder
84+
stackMarshaller deploy.StackMarshaller
85+
stackDeployer deploy.StackDeployer
86+
logger logr.Logger
87+
metricsCollector lbcmetrics.MetricCollector
8488

8589
maxConcurrentReconciles int
8690
}
@@ -94,22 +98,26 @@ func (r *serviceReconciler) Reconcile(ctx context.Context, req reconcile.Request
9498
}
9599

96100
func (r *serviceReconciler) reconcile(ctx context.Context, req reconcile.Request) error {
101+
defer r.metricsCollector.ObserveControllerReconcileLatency("service", "fetch service")
97102
svc := &corev1.Service{}
98103
if err := r.k8sClient.Get(ctx, req.NamespacedName, svc); err != nil {
99104
return client.IgnoreNotFound(err)
100105
}
106+
defer r.metricsCollector.ObserveControllerReconcileLatency("service", "build model")
101107
stack, lb, backendSGRequired, err := r.buildModel(ctx, svc)
102108
if err != nil {
103109
return err
104110
}
111+
112+
defer r.metricsCollector.ObserveControllerReconcileLatency("service", "clean up resources")
105113
if lb == nil {
106114
return r.cleanupLoadBalancerResources(ctx, svc, stack)
107115
}
108116
return r.reconcileLoadBalancerResources(ctx, svc, stack, lb, backendSGRequired)
109117
}
110118

111119
func (r *serviceReconciler) buildModel(ctx context.Context, svc *corev1.Service) (core.Stack, *elbv2model.LoadBalancer, bool, error) {
112-
stack, lb, backendSGRequired, err := r.modelBuilder.Build(ctx, svc)
120+
stack, lb, backendSGRequired, err := r.modelBuilder.Build(ctx, svc, r.metricsCollector)
113121
if err != nil {
114122
r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err))
115123
return nil, nil, false, err
@@ -124,7 +132,7 @@ func (r *serviceReconciler) buildModel(ctx context.Context, svc *corev1.Service)
124132
}
125133

126134
func (r *serviceReconciler) deployModel(ctx context.Context, svc *corev1.Service, stack core.Stack) error {
127-
if err := r.stackDeployer.Deploy(ctx, stack); err != nil {
135+
if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector); err != nil {
128136
var requeueNeededAfter *runtime.RequeueNeededAfter
129137
if errors.As(err, &requeueNeededAfter) {
130138
return err
@@ -139,25 +147,33 @@ func (r *serviceReconciler) deployModel(ctx context.Context, svc *corev1.Service
139147

140148
func (r *serviceReconciler) reconcileLoadBalancerResources(ctx context.Context, svc *corev1.Service, stack core.Stack,
141149
lb *elbv2model.LoadBalancer, backendSGRequired bool) error {
150+
151+
defer r.metricsCollector.ObserveControllerReconcileLatency("service", "add finalizers")
142152
if err := r.finalizerManager.AddFinalizers(ctx, svc, serviceFinalizer); err != nil {
143153
r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedAddFinalizer, fmt.Sprintf("Failed add finalizer due to %v", err))
144154
return err
145155
}
156+
defer r.metricsCollector.ObserveControllerReconcileLatency("service", "deploy model")
146157
err := r.deployModel(ctx, svc, stack)
147158
if err != nil {
159+
r.metricsCollector.ObserveControllerReconcileError("service", "deploy model error", "deploy model error")
148160
return err
149161
}
162+
defer r.metricsCollector.ObserveControllerReconcileLatency("service", "DNS resolve")
150163
lbDNS, err := lb.DNSName().Resolve(ctx)
151164
if err != nil {
165+
r.metricsCollector.ObserveControllerReconcileError("service", "deploy model error", "DNS resolve error")
152166
return err
153167
}
154168

155169
if !backendSGRequired {
156170
if err := r.backendSGProvider.Release(ctx, networking.ResourceTypeService, []types.NamespacedName{k8s.NamespacedName(svc)}); err != nil {
171+
r.metricsCollector.ObserveControllerReconcileError("service", "deploy model error", "release auto-generated backend SG error")
157172
return err
158173
}
159174
}
160175

176+
defer r.metricsCollector.ObserveControllerReconcileLatency("service", "update status")
161177
if err = r.updateServiceStatus(ctx, lbDNS, svc); err != nil {
162178
r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedUpdateStatus, fmt.Sprintf("Failed update status due to %v", err))
163179
return err

0 commit comments

Comments
 (0)