Optimizations for controller memory usage

cmd/manager/main.go

@@ -38,11 +38,13 @@ import (
 
 	"github.com/kubernetes-csi/csi-lib-utils/standardflags"
 	"go.uber.org/zap/zapcore"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -178,6 +180,19 @@ func main() {
 			Port:    9443,
 			TLSOpts: tlsOpts,
 		}),
+		Client: client.Options{
+			Cache: &client.CacheOptions{
+				DisableFor: []client.Object{
+					// Pods are fetched by the CSIAddonsNode controller
+					// Since we do not do this frequently the cache for it can be disabled
+					// This benefits us a lot as there can be a large number of pods that are present in the cache
+					&corev1.Pod{},
+					// Namespaces are fetched by the old PVC reconciler
+					// TODO: Remove this when the reconciler is phased out
+					&corev1.Namespace{},
+				},
+			},
+		},
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
@@ -223,14 +238,26 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "ReclaimSpaceCronJob")
 		os.Exit(1)
 	}
-	if err = (&controllers.PersistentVolumeClaimReconciler{
-		Client:             mgr.GetClient(),
-		Scheme:             mgr.GetScheme(),
-		ConnPool:           connPool,
-		SchedulePrecedence: cfg.SchedulePrecedence,
-	}).SetupWithManager(mgr, ctrlOptions); err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "PersistentVolumeClaim")
-		os.Exit(1)
+	if cfg.SchedulePrecedence == util.ScheduleSC {
+		setupLog.Info("Using new PVC controller for precedence", "schedulePrecedence", cfg.SchedulePrecedence)
+		if err = (&controllers.PVCReconiler{
+			Client:   mgr.GetClient(),
+			Scheme:   mgr.GetScheme(),
+			ConnPool: connPool,
+		}).SetupWithManager(mgr, ctrlOptions); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "PersistentVolumeClaim")
+			os.Exit(1)
+		}
+	} else {
+		if err = (&controllers.PersistentVolumeClaimReconciler{
+			Client:             mgr.GetClient(),
+			Scheme:             mgr.GetScheme(),
+			ConnPool:           connPool,
+			SchedulePrecedence: cfg.SchedulePrecedence,
+		}).SetupWithManager(mgr, ctrlOptions); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "PersistentVolumeClaim")
+			os.Exit(1)
+		}
 	}
 	if err = (&replicationController.VolumeReplicationReconciler{
 		Client:   mgr.GetClient(),

internal/controller/csiaddons/encryptionkeyrotationcronjob_controller.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	csiaddonsv1alpha1 "github.com/csi-addons/kubernetes-csi-addons/api/csiaddons/v1alpha1"
+	"github.com/csi-addons/kubernetes-csi-addons/internal/controller/utils"
 
 	"github.com/go-logr/logr"
 	"github.com/robfig/cron/v3"
@@ -72,10 +73,10 @@ func (r *EncryptionKeyRotationCronJobReconciler) Reconcile(ctx context.Context,
 
 	// Set default values for the optionals
 	if krcJob.Spec.FailedJobsHistoryLimit == nil {
-		*krcJob.Spec.FailedJobsHistoryLimit = defaultFailedJobsHistoryLimit
+		*krcJob.Spec.FailedJobsHistoryLimit = utils.DefaultFailedJobsHistoryLimit
 	}
 	if krcJob.Spec.SuccessfulJobsHistoryLimit == nil {
-		*krcJob.Spec.SuccessfulJobsHistoryLimit = defaultSuccessfulJobsHistoryLimit
+		*krcJob.Spec.SuccessfulJobsHistoryLimit = utils.DefaultSuccessfulJobsHistoryLimit
 	}
 
 	var childJobs csiaddonsv1alpha1.EncryptionKeyRotationJobList

internal/controller/csiaddons/encryptionkeyrotationjob_controller.go

@@ -24,6 +24,7 @@ import (
 
 	csiaddonsv1alpha1 "github.com/csi-addons/kubernetes-csi-addons/api/csiaddons/v1alpha1"
 	"github.com/csi-addons/kubernetes-csi-addons/internal/connection"
+	"github.com/csi-addons/kubernetes-csi-addons/internal/controller/utils"
 	"github.com/csi-addons/kubernetes-csi-addons/internal/proto"
 	"github.com/csi-addons/kubernetes-csi-addons/internal/util"
 	"github.com/csi-addons/spec/lib/go/identity"
@@ -38,8 +39,10 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 )
@@ -333,6 +336,12 @@ func (r *EncryptionKeyRotationJobReconciler) SetupWithManager(mgr ctrl.Manager,
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&csiaddonsv1alpha1.EncryptionKeyRotationJob{}).
 		WithEventFilter(predicate.GenerationChangedPredicate{}).
+		// This is to avoid "stop-the-world" events and wait for cache sync when we list VA
+		Watches(
+			&scv1.VolumeAttachment{},
+			&handler.EnqueueRequestForObject{},
+			builder.WithPredicates(utils.SilentPredicate()),
+		).
 		WithOptions(ctrlOptions).
 		Complete(r)
 }