improve clickhouse config

improve clickhouse config

Author: nicktrn

hosting/k8s/helm/templates/clickhouse.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.clickhouse.enabled }}
+{{- if and .Values.clickhouse.enabled (not .Values.clickhouse.external) }}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -34,8 +34,11 @@ spec:
           image: "{{ .Values.clickhouse.image.registry }}/{{ .Values.clickhouse.image.repository }}:{{ .Values.clickhouse.image.tag }}"
           imagePullPolicy: {{ .Values.clickhouse.image.pullPolicy }}
           ports:
+            - name: native
+              containerPort: {{ .Values.clickhouse.service.ports.native }}
+              protocol: TCP
             - name: http
-              containerPort: {{ .Values.clickhouse.service.targetPort }}
+              containerPort: {{ .Values.clickhouse.service.ports.http }}
               protocol: TCP
           env:
             - name: CLICKHOUSE_ADMIN_USER
@@ -52,7 +55,7 @@ spec:
                 - --host
                 - localhost
                 - --port
-                - "9000"
+                - {{ .Values.clickhouse.service.ports.native | quote }}
                 - --user
                 - {{ .Values.clickhouse.auth.adminUser }}
                 - --password
@@ -70,7 +73,7 @@ spec:
                 - --host
                 - localhost
                 - --port
-                - "9000"
+                - {{ .Values.clickhouse.service.ports.native | quote }}
                 - --user
                 - {{ .Values.clickhouse.auth.adminUser }}
                 - --password
@@ -128,8 +131,12 @@ spec:
   type: ClusterIP
   clusterIP: None
   ports:
+    - name: native
+      port: {{ .Values.clickhouse.service.ports.native }}
+      targetPort: native
+      protocol: TCP
     - name: http
-      port: {{ .Values.clickhouse.service.port }}
+      port: {{ .Values.clickhouse.service.ports.http }}
       targetPort: http
       protocol: TCP
   selector:
@@ -145,8 +152,12 @@ metadata:
 spec:
   type: {{ .Values.clickhouse.service.type }}
   ports:
+    - name: native
+      port: {{ .Values.clickhouse.service.ports.native }}
+      targetPort: native
+      protocol: TCP
     - name: http
-      port: {{ .Values.clickhouse.service.port }}
+      port: {{ .Values.clickhouse.service.ports.http }}
       targetPort: http
       protocol: TCP
   selector:

hosting/k8s/helm/templates/configmap.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.clickhouse.enabled }}
+{{- if and .Values.clickhouse.enabled (not .Values.clickhouse.external) }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

hosting/k8s/helm/templates/tests/test-clickhouse.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.clickhouse.enabled }}
+{{- if and .Values.clickhouse.enabled (not .Values.clickhouse.external) }}
 apiVersion: v1
 kind: Pod
 metadata:
@@ -16,6 +16,6 @@ spec:
       args:
         - |
           echo "Testing ClickHouse HTTP interface..."
-          curl -f "http://{{ .Values.clickhouse.auth.adminUser }}:{{ .Values.clickhouse.auth.adminPassword }}@{{ include "trigger-v4.fullname" . }}-clickhouse:{{ .Values.clickhouse.service.port }}/ping"
+          curl -f "http://{{ .Values.clickhouse.auth.adminUser }}:{{ .Values.clickhouse.auth.adminPassword }}@{{ include "trigger-v4.fullname" . }}-clickhouse:{{ .Values.clickhouse.service.ports.http }}/ping"
           echo "ClickHouse test completed successfully"
 {{- end }}

hosting/k8s/helm/templates/webapp.yaml

@@ -187,6 +187,20 @@ spec:
               value: {{ .Values.webapp.internal.otel.metric.exporterEnabled | quote }}
             - name: INTERNAL_OTEL_METRIC_EXPORTER_INTERVAL_MS
               value: {{ .Values.webapp.internal.otel.metric.exporterIntervalMs | quote }}
+            {{- if .Values.webapp.clickhouse.enabled }}
+            - name: CLICKHOUSE_URL
+              value: {{ if .Values.clickhouse.external }}{{ .Values.clickhouse.externalConnection.httpUrl | quote }}{{ else }}"http://{{ .Values.clickhouse.auth.adminUser }}:{{ .Values.clickhouse.auth.adminPassword }}@{{ include "trigger-v4.fullname" . }}-clickhouse:{{ .Values.clickhouse.service.ports.http }}"{{ end }}
+            - name: CLICKHOUSE_LOG_LEVEL
+              value: {{ .Values.webapp.clickhouse.logLevel | quote }}
+            {{- end }}
+            {{- if .Values.webapp.runReplication.enabled }}
+            - name: RUN_REPLICATION_ENABLED
+              value: "1"
+            - name: RUN_REPLICATION_CLICKHOUSE_URL
+              value: {{ if .Values.clickhouse.external }}{{ .Values.clickhouse.externalConnection.httpUrl | quote }}{{ else }}"http://{{ .Values.clickhouse.auth.adminUser }}:{{ .Values.clickhouse.auth.adminPassword }}@{{ include "trigger-v4.fullname" . }}-clickhouse:{{ .Values.clickhouse.service.ports.http }}"{{ end }}
+            - name: RUN_REPLICATION_LOG_LEVEL
+              value: {{ .Values.webapp.runReplication.logLevel | quote }}
+            {{- end }}
             {{- with .Values.webapp.extraEnv }}
             {{- toYaml . | nindent 12 }}
             {{- end }}

hosting/k8s/helm/values-production-example.yaml

@@ -46,19 +46,6 @@ webapp:
     requests:
       cpu: 1000m
       memory: 2Gi
-  # ClickHouse configuration (experimental)
-  extraEnv:
-    []
-    # - name: RUN_REPLICATION_CLICKHOUSE_URL
-    #   value: "http://clickhouse:9000"
-    # - name: RUN_REPLICATION_ENABLED
-    #   value: "1"
-    # - name: RUN_REPLICATION_LOG_LEVEL
-    #   value: "info"
-    # - name: CLICKHOUSE_URL
-    #   value: "http://clickhouse:9000"
-    # - name: CLICKHOUSE_LOG_LEVEL
-    #   value: "info"
 
 # Production PostgreSQL (or use external)
 postgres:

hosting/k8s/helm/values.yaml

@@ -95,17 +95,6 @@ webapp:
     #     secretKeyRef:
     #       name: my-secret
     #       key: secret-key
-    # ClickHouse configuration (experimental)
-    # - name: RUN_REPLICATION_CLICKHOUSE_URL
-    #   value: "http://clickhouse:9000"
-    # - name: RUN_REPLICATION_ENABLED
-    #   value: "0"
-    # - name: RUN_REPLICATION_LOG_LEVEL
-    #   value: "info"
-    # - name: CLICKHOUSE_URL
-    #   value: "http://clickhouse:9000"
-    # - name: CLICKHOUSE_LOG_LEVEL
-    #   value: "info"
 
   # ServiceMonitor for Prometheus monitoring
   serviceMonitor:
@@ -115,6 +104,18 @@ webapp:
     labels: {}
     basicAuth: {}
 
+  # ClickHouse integration (experimental)
+  # Usage patterns:
+  #   1. Internal ClickHouse: Set clickhouse.enabled=true, clickhouse.external=false, webapp.clickhouse.enabled=true
+  #   2. External ClickHouse: Set clickhouse.enabled=true, clickhouse.external=true, configure externalConnection URLs, webapp.clickhouse.enabled=true
+  #   3. No ClickHouse: Leave webapp.clickhouse.enabled=false (default)
+  clickhouse:
+    enabled: true
+    logLevel: "info" # one of: log, error, warn, info, debug
+  runReplication:
+    enabled: true
+    logLevel: "info" # one of: log, error, warn, info, debug
+
   # OpenTelemetry configuration
   internal:
     otel:
@@ -152,13 +153,13 @@ secrets:
   # If empty, a secret will be created with the values below
   # The secret must contain the following keys:
   #   - session-secret
-  #   - magic-link-secret  
+  #   - magic-link-secret
   #   - encryption-key
   #   - managed-worker-secret
   #   - object-store-access-key-id
   #   - object-store-secret-access-key
   existingSecret: ""
-  
+
   # Session secret for user authentication (32 hex chars)
   sessionSecret: "2818143646516f6fffd707b36f334bbb"
   # Magic link secret for passwordless login (32 hex chars)
@@ -301,6 +302,7 @@ electric:
 # ClickHouse configuration
 clickhouse:
   enabled: true
+  external: false
   image:
     registry: docker.io
     repository: bitnami/clickhouse
@@ -324,9 +326,14 @@ clickhouse:
     size: 10Gi
   service:
     type: ClusterIP
-    port: 9000
-    targetPort: 9000
+    ports:
+      native: 9000
+      http: 8123
   resources: {}
+  # External ClickHouse connection (when external: true)
+  externalConnection:
+    # HTTP interface URL (port 8123) - used by webapp for replication
+    httpUrl: ""
   # Extra environment variables for ClickHouse
   extraEnv:
     []