diff --git a/k8s/app/deployment.yaml b/k8s/app/deployment.yaml
index a9e6fdd..8d51ed2 100644
--- a/k8s/app/deployment.yaml
+++ b/k8s/app/deployment.yaml
@@ -11,10 +11,14 @@ spec:
metadata:
labels:
app: camera-onboarding
+ annotations:
+ prometheus.io/scrape: "true"
+ prometheus.io/port: "8080"
+ prometheus.io/path: "/actuator/prometheus"
spec:
containers:
- name: camera-onboarding
- image: mrsert/camera-onboarding:62417cc
+ image: mrsert/camera-onboarding:monitoring
resources:
requests:
cpu: "275m"
diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml
index a8b602c..9db16d5 100644
--- a/k8s/kustomization.yaml
+++ b/k8s/kustomization.yaml
@@ -1,4 +1,7 @@
# k8s/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
resources:
- - app
- - ../../../../../logging
\ No newline at end of file
+ - ./app
+ - ./logging
+ - ./monitoring
\ No newline at end of file
diff --git a/k8s/logging/grafana/dashboard-providers.yaml b/k8s/logging/grafana/dashboard-providers.yaml
new file mode 100644
index 0000000..716213f
--- /dev/null
+++ b/k8s/logging/grafana/dashboard-providers.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: grafana-dashboard-providers
+ namespace: logging
+data:
+ dashboards.yaml: |
+ apiVersion: 1
+ providers:
+ - name: 'spring-boot'
+ orgId: 1
+ folder: 'Spring Boot'
+ type: file
+ disableDeletion: false
+ updateIntervalSeconds: 10
+ options:
+ path: /var/lib/grafana/dashboards
\ No newline at end of file
diff --git a/k8s/logging/grafana/dashboard.yaml b/k8s/logging/grafana/dashboard.yaml
new file mode 100644
index 0000000..e190422
--- /dev/null
+++ b/k8s/logging/grafana/dashboard.yaml
@@ -0,0 +1,171 @@
+# k8s/logging/grafana/dashboard.yaml
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: grafana-spring-boot-dashboard
+ namespace: logging
+data:
+ spring-boot-dashboard.json: |
+ {
+ "id": null,
+ "title": "Spring Boot Metrics",
+ "tags": [
+ "spring-boot",
+ "micrometer"],
+ "style": "dark",
+ "timezone": "browser",
+ "panels": [
+ {
+ "id": 1,
+ "title": "JVM Memory Usage",
+ "type": "stat",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "jvm_memory_used_bytes{area=\"heap\", kubernetes_namespace=\"app\"}",
+ "legendFormat": "Heap Used"
+ },
+ {
+ "expr": "jvm_memory_max_bytes{area=\"heap\", kubernetes_namespace=\"app\"}",
+ "legendFormat": "Heap Max"
+ }
+ ],
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 0},
+ "fieldConfig": {
+ "defaults": {
+ "unit": "bytes",
+ "color": {
+ "mode": "palette-classic"}
+ }
+ }
+ },
+ {
+ "id": 2,
+ "title": "HTTP Requests Rate",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "rate(http_server_requests_seconds_count{kubernetes_namespace=\"app\"}[1m])",
+ "legendFormat": "{{method}} {{uri}}"
+ }
+ ],
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 0},
+ "fieldConfig": {
+ "defaults": {
+ "unit": "reqps",
+ "color": {
+ "mode": "palette-classic"}
+ }
+ }
+ },
+ {
+ "id": 3,
+ "title": "JVM Threads",
+ "type": "stat",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "jvm_threads_live_threads{kubernetes_namespace=\"app\"}",
+ "legendFormat": "Live Threads"
+ },
+ {
+ "expr": "jvm_threads_daemon_threads{kubernetes_namespace=\"app\"}",
+ "legendFormat": "Daemon Threads"
+ }
+ ],
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 8},
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "color": {
+ "mode": "palette-classic"}
+ }
+ }
+ },
+ {
+ "id": 4,
+ "title": "GC Activity",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "rate(jvm_gc_collection_seconds_count{kubernetes_namespace=\"app\"}[1m])",
+ "legendFormat": "{{gc}} GC Rate"
+ }
+ ],
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 8},
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ops",
+ "color": {
+ "mode": "palette-classic"}
+ }
+ }
+ },
+ {
+ "id": 5,
+ "title": "HTTP Response Times",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.95, rate(http_server_requests_seconds_bucket{kubernetes_namespace=\"app\"}[1m]))",
+ "legendFormat": "95th percentile"
+ },
+ {
+ "expr": "histogram_quantile(0.50, rate(http_server_requests_seconds_bucket{kubernetes_namespace=\"app\"}[1m]))",
+ "legendFormat": "50th percentile"
+ }
+ ],
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 16},
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s",
+ "color": {
+ "mode": "palette-classic"}
+ }
+ }
+ }
+ ],
+ "time": {
+ "from": "now-1h",
+ "to": "now"},
+ "refresh": "5s"
+ }
\ No newline at end of file
diff --git a/k8s/logging/grafana/grafana-configmap.yaml b/k8s/logging/grafana/grafana-configmap.yaml
index 6f12a7c..31c6c9b 100644
--- a/k8s/logging/grafana/grafana-configmap.yaml
+++ b/k8s/logging/grafana/grafana-configmap.yaml
@@ -11,5 +11,13 @@ data:
type: loki
access: proxy
url: http://loki:3100
+ isDefault: false
+ editable: true
+ - name: Prometheus
+ type: prometheus
+ access: proxy
+ url: http://prometheus-service:9090
isDefault: true
- editable: true
\ No newline at end of file
+ editable: true
+ readOnly: false
+ version: 1
\ No newline at end of file
diff --git a/k8s/logging/grafana/grafana-deployment.yaml b/k8s/logging/grafana/grafana-deployment.yaml
index c5ffa68..1df4c47 100644
--- a/k8s/logging/grafana/grafana-deployment.yaml
+++ b/k8s/logging/grafana/grafana-deployment.yaml
@@ -40,6 +40,10 @@ spec:
mountPath: /var/lib/grafana
- name: datasources
mountPath: /etc/grafana/provisioning/datasources
+ - name: dashboard-providers
+ mountPath: /etc/grafana/provisioning/dashboards
+ - name: spring-boot-dashboards
+ mountPath: /var/lib/grafana/dashboards
resources:
requests:
cpu: 100m
@@ -65,4 +69,10 @@ spec:
claimName: grafana-storage
- name: datasources
configMap:
- name: grafana-datasources
\ No newline at end of file
+ name: grafana-datasources
+ - name: dashboard-providers
+ configMap:
+ name: grafana-dashboard-providers
+ - name: spring-boot-dashboards
+ configMap:
+ name: grafana-spring-boot-dashboard
\ No newline at end of file
diff --git a/k8s/logging/kustomization.yaml b/k8s/logging/kustomization.yaml
index d286410..8689c86 100644
--- a/k8s/logging/kustomization.yaml
+++ b/k8s/logging/kustomization.yaml
@@ -1,14 +1,16 @@
# k8s/logging/kustomization.yaml
resources:
+ - logging-namespace.yaml
- grafana/grafana-configmap.yaml
- grafana/grafana-deployment.yaml
- grafana/grafana-pvc.yaml
- grafana/grafana-service.yaml
+ - grafana/dashboard.yaml
+ - grafana/dashboard-providers.yaml
- loki/loki-configmap.yaml
- loki/loki-deployment.yaml
- loki/loki-pvc.yaml
- loki/loki-service.yaml
- promtail/promtail-configmap.yaml
- promtail/promtail-daemonset.yaml
- - promtail/promtail-rbac.yaml
- - logging-namespace.yaml
\ No newline at end of file
+ - promtail/promtail-rbac.yaml
\ No newline at end of file
diff --git a/k8s/monitoring/kustomization.yaml b/k8s/monitoring/kustomization.yaml
new file mode 100644
index 0000000..d0e19da
--- /dev/null
+++ b/k8s/monitoring/kustomization.yaml
@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: logging
+resources:
+ - prometheus-configmap.yaml
+ - prometheus-rbac.yaml
+ - prometheus-service.yaml
+ - prometheus-statefulset.yaml
\ No newline at end of file
diff --git a/k8s/monitoring/prometheus-configmap.yaml b/k8s/monitoring/prometheus-configmap.yaml
new file mode 100644
index 0000000..fc541bb
--- /dev/null
+++ b/k8s/monitoring/prometheus-configmap.yaml
@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: prometheus-config
+ namespace: logging
+data:
+ prometheus.yml: |
+ global:
+ scrape_interval: 15s
+
+ scrape_configs:
+ # Kubernetes Service Discovery
+ - job_name: 'k8s-pods'
+ kubernetes_sd_configs:
+ - role: pod
+ relabel_configs:
+ - source_labels: [ __meta_kubernetes_pod_annotation_prometheus_io_scrape ]
+ action: keep
+ regex: true
+ - source_labels: [ __meta_kubernetes_pod_annotation_prometheus_io_path ]
+ action: replace
+ target_label: __metrics_path__
+ regex: (.+)
+ - source_labels: [ __address__, __meta_kubernetes_pod_annotation_prometheus_io_port ]
+ action: replace
+ regex: ([^:]+)(?::\d+)?;(\d+)
+ replacement: $1:$2
+ target_label: __address__
+ - action: labelmap
+ regex: __meta_kubernetes_pod_label_(.+)
+ - source_labels: [ __meta_kubernetes_namespace ]
+ action: replace
+ target_label: kubernetes_namespace
+ - source_labels: [ __meta_kubernetes_pod_name ]
+ action: replace
+ target_label: kubernetes_pod_name
+
+ # Prometheus self-monitoring
+ - job_name: 'prometheus'
+ static_configs:
+ - targets: [ 'localhost:9090' ]
\ No newline at end of file
diff --git a/k8s/monitoring/prometheus-rbac.yaml b/k8s/monitoring/prometheus-rbac.yaml
new file mode 100644
index 0000000..a093cb4
--- /dev/null
+++ b/k8s/monitoring/prometheus-rbac.yaml
@@ -0,0 +1,39 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: prometheus
+rules:
+ - apiGroups: [ "" ]
+ resources:
+ - nodes
+ - nodes/proxy
+ - services
+ - endpoints
+ - pods
+ verbs: [ "get", "list", "watch" ]
+ - apiGroups:
+ - extensions
+ resources:
+ - ingresses
+ verbs: [ "get", "list", "watch" ]
+ - nonResourceURLs: [ "/metrics" ]
+ verbs: [ "get" ]
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: prometheus-serviceaccount
+ namespace: logging
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: prometheus
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: prometheus
+subjects:
+ - kind: ServiceAccount
+ name: prometheus-serviceaccount
+ namespace: logging
\ No newline at end of file
diff --git a/k8s/monitoring/prometheus-service.yaml b/k8s/monitoring/prometheus-service.yaml
new file mode 100644
index 0000000..cf95f0d
--- /dev/null
+++ b/k8s/monitoring/prometheus-service.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: prometheus-service
+ namespace: logging
+ labels:
+ app: prometheus
+spec:
+ type: ClusterIP
+ ports:
+ - port: 9090
+ targetPort: 9090
+ name: http
+ selector:
+ app: prometheus
\ No newline at end of file
diff --git a/k8s/monitoring/prometheus-statefulset.yaml b/k8s/monitoring/prometheus-statefulset.yaml
new file mode 100644
index 0000000..c8190a7
--- /dev/null
+++ b/k8s/monitoring/prometheus-statefulset.yaml
@@ -0,0 +1,74 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: prometheus
+ namespace: logging
+ labels:
+ app: prometheus
+spec:
+ serviceName: prometheus
+ replicas: 1
+ updateStrategy:
+ type: RollingUpdate
+ selector:
+ matchLabels:
+ app: prometheus
+ template:
+ metadata:
+ labels:
+ app: prometheus
+ spec:
+ serviceAccountName: prometheus-serviceaccount
+ containers:
+ - name: prometheus
+ image: prom/prometheus:v2.45.0
+ args:
+ - '--storage.tsdb.retention=6h'
+ - '--storage.tsdb.path=/prometheus/'
+ - '--config.file=/etc/prometheus/prometheus.yml'
+ - '--web.enable-lifecycle'
+ ports:
+ - name: web
+ containerPort: 9090
+ readinessProbe:
+ httpGet:
+ path: /-/ready
+ port: 9090
+ initialDelaySeconds: 15
+ timeoutSeconds: 15
+ livenessProbe:
+ httpGet:
+ path: /-/healthy
+ port: 9090
+ initialDelaySeconds: 15
+ timeoutSeconds: 15
+ resources:
+ requests:
+ cpu: 200m
+ memory: 256Mi
+ limits:
+ cpu: 500m
+ memory: 512Mi
+ volumeMounts:
+ - name: config-volume
+ mountPath: /etc/prometheus
+ - name: prometheus-volume
+ mountPath: /prometheus
+ securityContext:
+ runAsNonRoot: true
+ runAsGroup: 65534
+ runAsUser: 65534
+ fsGroup: 65534
+ volumes:
+ - name: config-volume
+ configMap:
+ name: prometheus-config
+ volumeClaimTemplates:
+ - metadata:
+ name: prometheus-volume
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: "1Gi"
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 3deca5e..51566b9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -85,6 +85,11 @@
spring-boot-starter-actuator
+
+ io.micrometer
+ micrometer-registry-prometheus
+
+
com.microsoft.sqlserver
mssql-jdbc
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
index 0febd21..570ca71 100644
--- a/src/main/resources/application.properties
+++ b/src/main/resources/application.properties
@@ -15,6 +15,13 @@ spring.servlet.multipart.max-file-size=4MB
spring.servlet.multipart.max-request-size=40MB
spring.cloud.azure.profile.environment.storage-endpoint-suffix=core.windows.net
management.server.port=8080
-management.endpoints.web.exposure.include=health,info,metrics
+management.endpoints.web.exposure.include=health,info,metrics,prometheus
+management.endpoint.metrics.enabled=true
+management.endpoint.prometheus.enabled=true
+management.prometheus.metrics.export.enabled=true
+# JVM metrics
+management.metrics.enable.jvm=true
+management.metrics.enable.system=true
+management.metrics.enable.web=true
management.endpoint.health.probes.enabled=true
management.endpoint.health.show-details=always
\ No newline at end of file