Feat: Add servicemonitor for built-in detectors (#588)

* Add servicemonitor for built-in detectors

* Address review comments

Author: RobGeada

controllers/gorch/guardrailsorchestrator_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"github.com/go-logr/logr"
 	kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1"
+	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	"github.com/trustyai-explainability/trustyai-service-operator/controllers/metrics"
 	rbacv1 "k8s.io/api/rbac/v1"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -410,6 +411,34 @@ func (r *GuardrailsOrchestratorReconciler) Reconcile(ctx context.Context, req ct
 		}
 	}
 
+	existingSM := &monitoringv1.ServiceMonitor{}
+	err = r.Get(ctx, types.NamespacedName{Name: orchestrator.Name + "-service-monitor", Namespace: orchestrator.Namespace}, existingSM)
+	if orchestrator.Spec.EnableBuiltInDetectors {
+		if err != nil && errors.IsNotFound(err) {
+			// Define a new route
+			serviceMonitor := r.createServiceMonitor(ctx, orchestrator)
+			log.Info("Creating a new Service Monitor", "ServiceMonitor.Namespace", serviceMonitor.Namespace, "ServiceMonitor.Name", serviceMonitor.Name)
+			err = r.Create(ctx, serviceMonitor)
+			if err != nil {
+				r.handleReconciliationErrorWithTrace(ctx, log, orchestrator, err, ReconcileFailed, "Failed to create new ServiceMonitor", "ServiceMonitor.Namespace", serviceMonitor.Namespace, "ServiceMonitor.Name", serviceMonitor.Name)
+			}
+		} else if err != nil {
+			r.handleReconciliationError(ctx, log, orchestrator, err, ReconcileFailed, "Failed to get ServiceMonitor")
+			return ctrl.Result{}, err
+		}
+	} else {
+		if err == nil {
+			log.Info("Deleting ServiceMonitor because EnableBuiltInDetectors is false", "ServiceMonitor.Namespace", existingSM.Namespace, "ServiceMonitor.Name", existingSM.Name)
+			if delErr := r.Delete(ctx, existingSM); delErr != nil && !errors.IsNotFound(delErr) {
+				r.handleReconciliationErrorWithTrace(ctx, log, orchestrator, delErr, ReconcileFailed, "Failed to delete ServiceMonitor", "ServiceMonitor.Namespace", existingSM.Namespace, "ServiceMonitor.Name", existingSM.Name)
+				return ctrl.Result{}, delErr
+			}
+		} else if err != nil && !errors.IsNotFound(err) {
+			r.handleReconciliationError(ctx, log, orchestrator, err, ReconcileFailed, "Failed to get ServiceMonitor for deletion")
+			return ctrl.Result{}, err
+		}
+	}
+
 	// Finalize reconcilation
 	_, updateErr := r.reconcileStatuses(ctx, orchestrator)
 	if updateErr != nil {

controllers/gorch/servicemonitor.go

@@ -0,0 +1,32 @@
+package gorch
+
+import (
+	"context"
+	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	gorchv1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/gorch/v1alpha1"
+	templateParser "github.com/trustyai-explainability/trustyai-service-operator/controllers/gorch/templates"
+	"reflect"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+const serviceMonitorTemplatePath = "service-monitor.tmpl.yaml"
+
+type ServiceMonitorConfig struct {
+	Orchestrator gorchv1alpha1.GuardrailsOrchestrator
+}
+
+func (r *GuardrailsOrchestratorReconciler) createServiceMonitor(ctx context.Context, orchestrator *gorchv1alpha1.GuardrailsOrchestrator) *monitoringv1.ServiceMonitor {
+
+	serviceMonitorConfig := ServiceMonitorConfig{
+		Orchestrator: *orchestrator,
+	}
+
+	var serviceMonitor *monitoringv1.ServiceMonitor
+	serviceMonitor, err := templateParser.ParseResource[monitoringv1.ServiceMonitor](serviceMonitorTemplatePath, serviceMonitorConfig, reflect.TypeOf(&monitoringv1.ServiceMonitor{}))
+	if err != nil {
+		log.FromContext(ctx).Error(err, "Failed to parse service monitor template")
+	}
+	controllerutil.SetControllerReference(orchestrator, serviceMonitor, r.Scheme)
+	return serviceMonitor
+}

controllers/gorch/templates/service-monitor.tmpl.yaml

@@ -0,0 +1,27 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+    name: {{ .Orchestrator.Name }}-service-monitor
+    namespace: {{.Orchestrator.Namespace}}
+    labels:
+        component: {{ .Orchestrator.Name }}
+spec:
+    endpoints:
+        - bearerTokenSecret:
+              key: ''
+          honorLabels: true
+          interval: 30s
+          metricRelabelings:
+              - action: keep
+                regex: trustyai_guardrails_.*
+                sourceLabels:
+                    - __name__
+          path: /metrics
+          scheme: http
+          port: detector-metrics
+    selector:
+        matchLabels:
+            component: {{ .Orchestrator.Name }}
+    namespaceSelector:
+      matchNames:
+        - {{ .Orchestrator.Namespace }}

controllers/gorch/templates/service.tmpl.yaml

@@ -7,7 +7,8 @@ metadata:
     service.beta.openshift.io/serving-cert-secret-name: {{ .Orchestrator.Name }}-tls
   labels:
     app: {{.Orchestrator.Name}}
-    component: {{.Orchestrator.Name}}
+    component: {{ .Orchestrator.Name }}
+
 spec:
   ipFamilies:
     - IPv4
@@ -32,6 +33,12 @@ spec:
       protocol: TCP
       port: 8034
       targetPort: 8034
+    {{if .Orchestrator.Spec.EnableBuiltInDetectors}}
+    - name: detector-metrics
+      protocol: TCP
+      port: 8080
+      targetPort: 8080
+    {{end}}
   internalTrafficPolicy: Cluster
   type: ClusterIP
   ipFamilyPolicy: SingleStack