diff --git a/k8s/app/deployment.yaml b/k8s/app/deployment.yaml index a9e6fdd..8d51ed2 100644 --- a/k8s/app/deployment.yaml +++ b/k8s/app/deployment.yaml @@ -11,10 +11,14 @@ spec: metadata: labels: app: camera-onboarding + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + prometheus.io/path: "/actuator/prometheus" spec: containers: - name: camera-onboarding - image: mrsert/camera-onboarding:62417cc + image: mrsert/camera-onboarding:monitoring resources: requests: cpu: "275m" diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index a8b602c..9db16d5 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -1,4 +1,7 @@ # k8s/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization resources: - - app - - ../../../../../logging \ No newline at end of file + - ./app + - ./logging + - ./monitoring \ No newline at end of file diff --git a/k8s/logging/grafana/dashboard-providers.yaml b/k8s/logging/grafana/dashboard-providers.yaml new file mode 100644 index 0000000..716213f --- /dev/null +++ b/k8s/logging/grafana/dashboard-providers.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-providers + namespace: logging +data: + dashboards.yaml: | + apiVersion: 1 + providers: + - name: 'spring-boot' + orgId: 1 + folder: 'Spring Boot' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/k8s/logging/grafana/dashboard.yaml b/k8s/logging/grafana/dashboard.yaml new file mode 100644 index 0000000..e190422 --- /dev/null +++ b/k8s/logging/grafana/dashboard.yaml @@ -0,0 +1,171 @@ +# k8s/logging/grafana/dashboard.yaml + +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-spring-boot-dashboard + namespace: logging +data: + spring-boot-dashboard.json: | + { + "id": null, + "title": "Spring Boot Metrics", + "tags": [ + "spring-boot", + "micrometer"], + "style": "dark", + "timezone": "browser", + "panels": [ + { + "id": 1, + "title": "JVM Memory Usage", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "targets": [ + { + "expr": "jvm_memory_used_bytes{area=\"heap\", kubernetes_namespace=\"app\"}", + "legendFormat": "Heap Used" + }, + { + "expr": "jvm_memory_max_bytes{area=\"heap\", kubernetes_namespace=\"app\"}", + "legendFormat": "Heap Max" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0}, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "palette-classic"} + } + } + }, + { + "id": 2, + "title": "HTTP Requests Rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "targets": [ + { + "expr": "rate(http_server_requests_seconds_count{kubernetes_namespace=\"app\"}[1m])", + "legendFormat": "{{method}} {{uri}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0}, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { + "mode": "palette-classic"} + } + } + }, + { + "id": 3, + "title": "JVM Threads", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "targets": [ + { + "expr": "jvm_threads_live_threads{kubernetes_namespace=\"app\"}", + "legendFormat": "Live Threads" + }, + { + "expr": "jvm_threads_daemon_threads{kubernetes_namespace=\"app\"}", + "legendFormat": "Daemon Threads" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8}, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic"} + } + } + }, + { + "id": 4, + "title": "GC Activity", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "targets": [ + { + "expr": "rate(jvm_gc_collection_seconds_count{kubernetes_namespace=\"app\"}[1m])", + "legendFormat": "{{gc}} GC Rate" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8}, + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { + "mode": "palette-classic"} + } + } + }, + { + "id": 5, + "title": "HTTP Response Times", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, rate(http_server_requests_seconds_bucket{kubernetes_namespace=\"app\"}[1m]))", + "legendFormat": "95th percentile" + }, + { + "expr": "histogram_quantile(0.50, rate(http_server_requests_seconds_bucket{kubernetes_namespace=\"app\"}[1m]))", + "legendFormat": "50th percentile" + } + ], + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16}, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "palette-classic"} + } + } + } + ], + "time": { + "from": "now-1h", + "to": "now"}, + "refresh": "5s" + } \ No newline at end of file diff --git a/k8s/logging/grafana/grafana-configmap.yaml b/k8s/logging/grafana/grafana-configmap.yaml index 6f12a7c..31c6c9b 100644 --- a/k8s/logging/grafana/grafana-configmap.yaml +++ b/k8s/logging/grafana/grafana-configmap.yaml @@ -11,5 +11,13 @@ data: type: loki access: proxy url: http://loki:3100 + isDefault: false + editable: true + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus-service:9090 isDefault: true - editable: true \ No newline at end of file + editable: true + readOnly: false + version: 1 \ No newline at end of file diff --git a/k8s/logging/grafana/grafana-deployment.yaml b/k8s/logging/grafana/grafana-deployment.yaml index c5ffa68..1df4c47 100644 --- a/k8s/logging/grafana/grafana-deployment.yaml +++ b/k8s/logging/grafana/grafana-deployment.yaml @@ -40,6 +40,10 @@ spec: mountPath: /var/lib/grafana - name: datasources mountPath: /etc/grafana/provisioning/datasources + - name: dashboard-providers + mountPath: /etc/grafana/provisioning/dashboards + - name: spring-boot-dashboards + mountPath: /var/lib/grafana/dashboards resources: requests: cpu: 100m @@ -65,4 +69,10 @@ spec: claimName: grafana-storage - name: datasources configMap: - name: grafana-datasources \ No newline at end of file + name: grafana-datasources + - name: dashboard-providers + configMap: + name: grafana-dashboard-providers + - name: spring-boot-dashboards + configMap: + name: grafana-spring-boot-dashboard \ No newline at end of file diff --git a/k8s/logging/kustomization.yaml b/k8s/logging/kustomization.yaml index d286410..8689c86 100644 --- a/k8s/logging/kustomization.yaml +++ b/k8s/logging/kustomization.yaml @@ -1,14 +1,16 @@ # k8s/logging/kustomization.yaml resources: + - logging-namespace.yaml - grafana/grafana-configmap.yaml - grafana/grafana-deployment.yaml - grafana/grafana-pvc.yaml - grafana/grafana-service.yaml + - grafana/dashboard.yaml + - grafana/dashboard-providers.yaml - loki/loki-configmap.yaml - loki/loki-deployment.yaml - loki/loki-pvc.yaml - loki/loki-service.yaml - promtail/promtail-configmap.yaml - promtail/promtail-daemonset.yaml - - promtail/promtail-rbac.yaml - - logging-namespace.yaml \ No newline at end of file + - promtail/promtail-rbac.yaml \ No newline at end of file diff --git a/k8s/monitoring/kustomization.yaml b/k8s/monitoring/kustomization.yaml new file mode 100644 index 0000000..d0e19da --- /dev/null +++ b/k8s/monitoring/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: logging +resources: + - prometheus-configmap.yaml + - prometheus-rbac.yaml + - prometheus-service.yaml + - prometheus-statefulset.yaml \ No newline at end of file diff --git a/k8s/monitoring/prometheus-configmap.yaml b/k8s/monitoring/prometheus-configmap.yaml new file mode 100644 index 0000000..fc541bb --- /dev/null +++ b/k8s/monitoring/prometheus-configmap.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + namespace: logging +data: + prometheus.yml: | + global: + scrape_interval: 15s + + scrape_configs: + # Kubernetes Service Discovery + - job_name: 'k8s-pods' + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [ __meta_kubernetes_pod_annotation_prometheus_io_scrape ] + action: keep + regex: true + - source_labels: [ __meta_kubernetes_pod_annotation_prometheus_io_path ] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [ __address__, __meta_kubernetes_pod_annotation_prometheus_io_port ] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [ __meta_kubernetes_namespace ] + action: replace + target_label: kubernetes_namespace + - source_labels: [ __meta_kubernetes_pod_name ] + action: replace + target_label: kubernetes_pod_name + + # Prometheus self-monitoring + - job_name: 'prometheus' + static_configs: + - targets: [ 'localhost:9090' ] \ No newline at end of file diff --git a/k8s/monitoring/prometheus-rbac.yaml b/k8s/monitoring/prometheus-rbac.yaml new file mode 100644 index 0000000..a093cb4 --- /dev/null +++ b/k8s/monitoring/prometheus-rbac.yaml @@ -0,0 +1,39 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: + - apiGroups: [ "" ] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: [ "get", "list", "watch" ] + - apiGroups: + - extensions + resources: + - ingresses + verbs: [ "get", "list", "watch" ] + - nonResourceURLs: [ "/metrics" ] + verbs: [ "get" ] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-serviceaccount + namespace: logging +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus-serviceaccount + namespace: logging \ No newline at end of file diff --git a/k8s/monitoring/prometheus-service.yaml b/k8s/monitoring/prometheus-service.yaml new file mode 100644 index 0000000..cf95f0d --- /dev/null +++ b/k8s/monitoring/prometheus-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus-service + namespace: logging + labels: + app: prometheus +spec: + type: ClusterIP + ports: + - port: 9090 + targetPort: 9090 + name: http + selector: + app: prometheus \ No newline at end of file diff --git a/k8s/monitoring/prometheus-statefulset.yaml b/k8s/monitoring/prometheus-statefulset.yaml new file mode 100644 index 0000000..c8190a7 --- /dev/null +++ b/k8s/monitoring/prometheus-statefulset.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: prometheus + namespace: logging + labels: + app: prometheus +spec: + serviceName: prometheus + replicas: 1 + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + serviceAccountName: prometheus-serviceaccount + containers: + - name: prometheus + image: prom/prometheus:v2.45.0 + args: + - '--storage.tsdb.retention=6h' + - '--storage.tsdb.path=/prometheus/' + - '--config.file=/etc/prometheus/prometheus.yml' + - '--web.enable-lifecycle' + ports: + - name: web + containerPort: 9090 + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + initialDelaySeconds: 15 + timeoutSeconds: 15 + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + initialDelaySeconds: 15 + timeoutSeconds: 15 + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + volumeMounts: + - name: config-volume + mountPath: /etc/prometheus + - name: prometheus-volume + mountPath: /prometheus + securityContext: + runAsNonRoot: true + runAsGroup: 65534 + runAsUser: 65534 + fsGroup: 65534 + volumes: + - name: config-volume + configMap: + name: prometheus-config + volumeClaimTemplates: + - metadata: + name: prometheus-volume + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1Gi" \ No newline at end of file diff --git a/pom.xml b/pom.xml index 3deca5e..51566b9 100644 --- a/pom.xml +++ b/pom.xml @@ -85,6 +85,11 @@ spring-boot-starter-actuator + + io.micrometer + micrometer-registry-prometheus + + com.microsoft.sqlserver mssql-jdbc diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 0febd21..570ca71 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -15,6 +15,13 @@ spring.servlet.multipart.max-file-size=4MB spring.servlet.multipart.max-request-size=40MB spring.cloud.azure.profile.environment.storage-endpoint-suffix=core.windows.net management.server.port=8080 -management.endpoints.web.exposure.include=health,info,metrics +management.endpoints.web.exposure.include=health,info,metrics,prometheus +management.endpoint.metrics.enabled=true +management.endpoint.prometheus.enabled=true +management.prometheus.metrics.export.enabled=true +# JVM metrics +management.metrics.enable.jvm=true +management.metrics.enable.system=true +management.metrics.enable.web=true management.endpoint.health.probes.enabled=true management.endpoint.health.show-details=always \ No newline at end of file