Task/monitoring setup

k8s/app/deployment.yaml

@@ -11,10 +11,14 @@ spec:
     metadata:
       labels:
         app: camera-onboarding
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+        prometheus.io/path: "/actuator/prometheus"
     spec:
       containers:
         - name: camera-onboarding
-          image: mrsert/camera-onboarding:62417cc
+          image: mrsert/camera-onboarding:monitoring
           resources:
             requests:
               cpu: "275m"

k8s/kustomization.yaml

@@ -1,4 +1,7 @@
 # k8s/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
 resources:
-  - app
-  - ../../../../../logging
+  - ./app
+  - ./logging
+  - ./monitoring

k8s/logging/grafana/dashboard-providers.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-dashboard-providers
+  namespace: logging
+data:
+  dashboards.yaml: |
+    apiVersion: 1
+    providers:
+      - name: 'spring-boot'
+        orgId: 1
+        folder: 'Spring Boot'
+        type: file
+        disableDeletion: false
+        updateIntervalSeconds: 10
+        options:
+          path: /var/lib/grafana/dashboards

k8s/logging/grafana/dashboard.yaml

@@ -0,0 +1,171 @@
+# k8s/logging/grafana/dashboard.yaml
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-spring-boot-dashboard
+  namespace: logging
+data:
+  spring-boot-dashboard.json: |
+    {
+      "id": null,
+      "title": "Spring Boot Metrics",
+      "tags": [
+        "spring-boot",
+        "micrometer"],
+      "style": "dark",
+      "timezone": "browser",
+      "panels": [
+        {
+          "id": 1,
+          "title": "JVM Memory Usage",
+          "type": "stat",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "Prometheus"
+          },
+          "targets": [
+            {
+              "expr": "jvm_memory_used_bytes{area=\"heap\", kubernetes_namespace=\"app\"}",
+              "legendFormat": "Heap Used"
+            },
+            {
+              "expr": "jvm_memory_max_bytes{area=\"heap\", kubernetes_namespace=\"app\"}",
+              "legendFormat": "Heap Max"
+            }
+          ],
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 0},
+          "fieldConfig": {
+            "defaults": {
+              "unit": "bytes",
+              "color": {
+                "mode": "palette-classic"}
+            }
+          }
+        },
+        {
+          "id": 2,
+          "title": "HTTP Requests Rate",
+          "type": "timeseries",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "Prometheus"
+          },
+          "targets": [
+            {
+              "expr": "rate(http_server_requests_seconds_count{kubernetes_namespace=\"app\"}[1m])",
+              "legendFormat": "{{method}} {{uri}}"
+            }
+          ],
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 0},
+          "fieldConfig": {
+            "defaults": {
+              "unit": "reqps",
+              "color": {
+                "mode": "palette-classic"}
+            }
+          }
+        },
+        {
+          "id": 3,
+          "title": "JVM Threads",
+          "type": "stat",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "Prometheus"
+          },
+          "targets": [
+            {
+              "expr": "jvm_threads_live_threads{kubernetes_namespace=\"app\"}",
+              "legendFormat": "Live Threads"
+            },
+            {
+              "expr": "jvm_threads_daemon_threads{kubernetes_namespace=\"app\"}",
+              "legendFormat": "Daemon Threads"
+            }
+          ],
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 8},
+          "fieldConfig": {
+            "defaults": {
+              "unit": "short",
+              "color": {
+                "mode": "palette-classic"}
+            }
+          }
+        },
+        {
+          "id": 4,
+          "title": "GC Activity",
+          "type": "timeseries",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "Prometheus"
+          },
+          "targets": [
+            {
+              "expr": "rate(jvm_gc_collection_seconds_count{kubernetes_namespace=\"app\"}[1m])",
+              "legendFormat": "{{gc}} GC Rate"
+            }
+          ],
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 8},
+          "fieldConfig": {
+            "defaults": {
+              "unit": "ops",
+              "color": {
+                "mode": "palette-classic"}
+            }
+          }
+        },
+        {
+          "id": 5,
+          "title": "HTTP Response Times",
+          "type": "timeseries",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "Prometheus"
+          },
+          "targets": [
+            {
+              "expr": "histogram_quantile(0.95, rate(http_server_requests_seconds_bucket{kubernetes_namespace=\"app\"}[1m]))",
+              "legendFormat": "95th percentile"
+            },
+            {
+              "expr": "histogram_quantile(0.50, rate(http_server_requests_seconds_bucket{kubernetes_namespace=\"app\"}[1m]))",
+              "legendFormat": "50th percentile"
+            }
+          ],
+          "gridPos": {
+            "h": 8,
+            "w": 24,
+            "x": 0,
+            "y": 16},
+          "fieldConfig": {
+            "defaults": {
+              "unit": "s",
+              "color": {
+                "mode": "palette-classic"}
+            }
+          }
+        }
+      ],
+      "time": {
+        "from": "now-1h",
+        "to": "now"},
+      "refresh": "5s"
+    }

k8s/logging/grafana/grafana-configmap.yaml

@@ -11,5 +11,13 @@ data:
         type: loki
         access: proxy
         url: http://loki:3100
+        isDefault: false
+        editable: true
+      - name: Prometheus
+        type: prometheus
+        access: proxy
+        url: http://prometheus-service:9090
         isDefault: true
-        editable: true
+        editable: true
+        readOnly: false
+        version: 1

k8s/logging/grafana/grafana-deployment.yaml

@@ -40,6 +40,10 @@ spec:
               mountPath: /var/lib/grafana
             - name: datasources
               mountPath: /etc/grafana/provisioning/datasources
+            - name: dashboard-providers
+              mountPath: /etc/grafana/provisioning/dashboards
+            - name: spring-boot-dashboards
+              mountPath: /var/lib/grafana/dashboards
           resources:
             requests:
               cpu: 100m
@@ -65,4 +69,10 @@ spec:
             claimName: grafana-storage
         - name: datasources
           configMap:
-            name: grafana-datasources
+            name: grafana-datasources
+        - name: dashboard-providers
+          configMap:
+            name: grafana-dashboard-providers
+        - name: spring-boot-dashboards
+          configMap:
+            name: grafana-spring-boot-dashboard

k8s/logging/kustomization.yaml

@@ -1,14 +1,16 @@
 # k8s/logging/kustomization.yaml
 resources:
+  - logging-namespace.yaml
   - grafana/grafana-configmap.yaml
   - grafana/grafana-deployment.yaml
   - grafana/grafana-pvc.yaml
   - grafana/grafana-service.yaml
+  - grafana/dashboard.yaml
+  - grafana/dashboard-providers.yaml
   - loki/loki-configmap.yaml
   - loki/loki-deployment.yaml
   - loki/loki-pvc.yaml
   - loki/loki-service.yaml
   - promtail/promtail-configmap.yaml
   - promtail/promtail-daemonset.yaml
-  - promtail/promtail-rbac.yaml
-  - logging-namespace.yaml
+  - promtail/promtail-rbac.yaml

k8s/monitoring/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: logging
+resources:
+  - prometheus-configmap.yaml
+  - prometheus-rbac.yaml
+  - prometheus-service.yaml
+  - prometheus-statefulset.yaml

k8s/monitoring/prometheus-configmap.yaml

@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-config
+  namespace: logging
+data:
+  prometheus.yml: |
+    global:
+      scrape_interval: 15s
+
+    scrape_configs:
+      # Kubernetes Service Discovery
+      - job_name: 'k8s-pods'
+        kubernetes_sd_configs:
+          - role: pod
+        relabel_configs:
+          - source_labels: [ __meta_kubernetes_pod_annotation_prometheus_io_scrape ]
+            action: keep
+            regex: true
+          - source_labels: [ __meta_kubernetes_pod_annotation_prometheus_io_path ]
+            action: replace
+            target_label: __metrics_path__
+            regex: (.+)
+          - source_labels: [ __address__, __meta_kubernetes_pod_annotation_prometheus_io_port ]
+            action: replace
+            regex: ([^:]+)(?::\d+)?;(\d+)
+            replacement: $1:$2
+            target_label: __address__
+          - action: labelmap
+            regex: __meta_kubernetes_pod_label_(.+)
+          - source_labels: [ __meta_kubernetes_namespace ]
+            action: replace
+            target_label: kubernetes_namespace
+          - source_labels: [ __meta_kubernetes_pod_name ]
+            action: replace
+            target_label: kubernetes_pod_name
+
+      # Prometheus self-monitoring
+      - job_name: 'prometheus'
+        static_configs:
+          - targets: [ 'localhost:9090' ]

k8s/monitoring/prometheus-rbac.yaml

@@ -0,0 +1,39 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus
+rules:
+  - apiGroups: [ "" ]
+    resources:
+      - nodes
+      - nodes/proxy
+      - services
+      - endpoints
+      - pods
+    verbs: [ "get", "list", "watch" ]
+  - apiGroups:
+      - extensions
+    resources:
+      - ingresses
+    verbs: [ "get", "list", "watch" ]
+  - nonResourceURLs: [ "/metrics" ]
+    verbs: [ "get" ]
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus-serviceaccount
+  namespace: logging
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus
+subjects:
+  - kind: ServiceAccount
+    name: prometheus-serviceaccount
+    namespace: logging

k8s/monitoring/prometheus-service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus-service
+  namespace: logging
+  labels:
+    app: prometheus
+spec:
+  type: ClusterIP
+  ports:
+    - port: 9090
+      targetPort: 9090
+      name: http
+  selector:
+    app: prometheus