K8s: Add configs for sessions external datastore

Signed-off-by: Viet Nguyen Duc <nguyenducviet4496@gmail.com>

Author: VietND96

Base/Dockerfile

@@ -11,6 +11,8 @@ ARG OPENTELEMETRY_VERSION=1.44.1
 ARG GRPC_VERSION=1.68.1
 ARG NETTY_VERSION=4.1.115.Final
 ARG CS_VERSION=2.1.18
+ARG POSTGRESQL_VERSION=42.7.4
+ARG MVN_SELENIUM_VERSION=4.27.0
 
 #Arguments to define the user running Selenium
 ARG SEL_USER=seluser
@@ -118,8 +120,16 @@ RUN --mount=type=secret,id=SEL_PASSWD \
   && if [ -f "/tmp/cs" ]; then \
         java -jar /tmp/cs fetch --classpath --cache /external_jars \
         io.opentelemetry:opentelemetry-exporter-otlp:${OPENTELEMETRY_VERSION} \
-        io.grpc:grpc-netty:${GRPC_VERSION} io.netty:netty-codec-http:${NETTY_VERSION} > /external_jars/.classpath.txt \
-        && chmod 664 /external_jars/.classpath.txt ; \
+        io.grpc:grpc-netty:${GRPC_VERSION} \
+        io.netty:netty-codec-http:${NETTY_VERSION} \
+        > /external_jars/.classpath.txt \
+        && chmod 664 /external_jars/.classpath.txt \
+        && java -jar /tmp/cs fetch --classpath --cache /external_jars \
+        org.seleniumhq.selenium:selenium-session-map-jdbc:${MVN_SELENIUM_VERSION} \
+        org.postgresql:postgresql:${POSTGRESQL_VERSION} \
+        org.seleniumhq.selenium:selenium-session-map-redis:${MVN_SELENIUM_VERSION} \
+        > /external_jars/.classpath_session_map.txt \
+        && chmod 664 /external_jars/.classpath_session_map.txt ; \
      fi \
   && rm -fr /root/.cache/* \
   # (Note that .bashrc is only executed in interactive bash shells.)

Makefile

@@ -917,7 +917,7 @@ chart_test_autoscaling_disabled:
 	./tests/charts/make/chart_test.sh NoAutoscaling
 
 chart_test_autoscaling_deployment_https:
-	PLATFORMS=$(PLATFORMS) CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_BASIC_AUTH=true \
+	PLATFORMS=$(PLATFORMS) CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_BASIC_AUTH=true TEST_EXTERNAL_DATASTORE=postgresql \
 	SECURE_INGRESS_ONLY_DEFAULT=true INGRESS_DISABLE_USE_HTTP2=true SELENIUM_GRID_PROTOCOL=https CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_PORT=443 \
 	SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=0 MAX_SESSIONS_FIREFOX=1 MAX_SESSIONS_EDGE=1 MAX_SESSIONS_CHROME=1 TEST_NAME_OVERRIDE=true \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \
@@ -955,7 +955,7 @@ chart_test_autoscaling_job:
 	./tests/charts/make/chart_test.sh JobAutoscaling
 
 chart_test_autoscaling_playwright_connect_grid:
-	PLATFORMS=$(PLATFORMS) CHART_ENABLE_TRACING=true CHART_ENABLE_BASIC_AUTH=true MATRIX_TESTS=CDPTests \
+	PLATFORMS=$(PLATFORMS) CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_BASIC_AUTH=true TEST_EXTERNAL_DATASTORE=redis MATRIX_TESTS=CDPTests \
 	BASIC_AUTH_USERNAME=docker-selenium BASIC_AUTH_PASSWORD=2NMI4jdBi6k7bENoeUfV25295VvzwAE9chM24a+2VL95uOHozo \
 	SECURE_INGRESS_ONLY_DEFAULT=true SECURE_USE_EXTERNAL_CERT=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -I | cut -d' ' -f1) SELENIUM_GRID_PORT=443 \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \

Sessions/Dockerfile

@@ -4,17 +4,24 @@ FROM ${NAMESPACE}/base:${VERSION}
 ARG AUTHORS
 LABEL authors=${AUTHORS}
 
-USER ${SEL_UID}
-
 #========================
 # Selenium SessionMap Configuration
 #========================
 
-EXPOSE 5556
+USER root
 
-COPY --chown="${SEL_UID}:${SEL_GID}" start-selenium-grid-sessions.sh \
+COPY --chown="${SEL_UID}:${SEL_GID}" start-selenium-grid-sessions.sh generate_config \
     /opt/bin/
+RUN chmod +x /opt/bin/start-selenium-grid-sessions.sh /opt/bin/generate_config
 
 COPY selenium-grid-sessions.conf /etc/supervisor/conf.d/
 
-ENV SE_OTEL_SERVICE_NAME="selenium-session-map"
+USER ${SEL_UID}
+
+EXPOSE 5556
+
+ENV SE_OTEL_SERVICE_NAME="selenium-session-map" \
+    # Path to the Configfile
+    CONFIG_FILE=/opt/selenium/config.toml \
+    GENERATE_CONFIG=true \
+    SE_SESSIONS_MAP_EXTERNAL_DATASTORE=false

Sessions/generate_config

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+if [[ -z "$CONFIG_FILE" ]]; then
+	FILENAME="/opt/selenium/config.toml"
+else
+	FILENAME="$CONFIG_FILE"
+fi
+
+echo "[sessions]" >"$FILENAME"
+
+if [ "${SE_SESSIONS_MAP_EXTERNAL_DATASTORE}" = "true" ]; then
+	if [[ -n "${SE_SESSIONS_MAP_EXTERNAL_SCHEME}" ]]; then
+		echo "scheme = \"${SE_SESSIONS_MAP_EXTERNAL_SCHEME}\"" >>"$FILENAME"
+	fi
+
+	if [[ -n "${SE_SESSIONS_MAP_EXTERNAL_IMPLEMENTATION}" ]]; then
+		echo "implementation = \"${SE_SESSIONS_MAP_EXTERNAL_IMPLEMENTATION}\"" >>"$FILENAME"
+	fi
+
+	if [[ -n "${SE_SESSIONS_MAP_EXTERNAL_HOSTNAME}" ]]; then
+		echo "hostname = \"${SE_SESSIONS_MAP_EXTERNAL_HOSTNAME}\"" >>"$FILENAME"
+	fi
+
+	if [[ -n "${SE_SESSIONS_MAP_EXTERNAL_PORT}" ]]; then
+		echo "port = \"${SE_SESSIONS_MAP_EXTERNAL_PORT}\"" >>"$FILENAME"
+	fi
+
+	if [[ -n "${SE_SESSIONS_MAP_EXTERNAL_JDBC_URL}" ]]; then
+		echo "jdbc-url = \"${SE_SESSIONS_MAP_EXTERNAL_JDBC_URL}\"" >>"$FILENAME"
+	fi
+
+	if [[ -n "${SE_SESSIONS_MAP_EXTERNAL_JDBC_USER}" ]]; then
+		echo "jdbc-user = \"${SE_SESSIONS_MAP_EXTERNAL_JDBC_USER}\"" >>"$FILENAME"
+	fi
+
+	if [[ -n "${SE_SESSIONS_MAP_EXTERNAL_JDBC_PASSWORD}" ]]; then
+		echo "jdbc-password = \"${SE_SESSIONS_MAP_EXTERNAL_JDBC_PASSWORD}\"" >>"$FILENAME"
+	fi
+fi

Sessions/start-selenium-grid-sessions.sh

@@ -3,8 +3,6 @@
 # set -e: exit asap if a command exits with a non-zero status
 set -e
 
-echo "Starting Selenium Grid Sessions..."
-
 function append_se_opts() {
   local option="${1}"
   local value="${2:-""}"
@@ -98,12 +96,21 @@ if [ ! -z "$SE_REGISTRATION_SECRET" ]; then
   append_se_opts "--registration-secret" "${SE_REGISTRATION_SECRET}" "false"
 fi
 
+if [ "$GENERATE_CONFIG" = true ]; then
+  echo "Generating Selenium Config for Sessions"
+  /opt/bin/generate_config
+fi
+
+if [ ! -z "${CONFIG_FILE}" ]; then
+  append_se_opts "--config" "${CONFIG_FILE}"
+fi
+
 EXTRA_LIBS=""
 
 if [ "$SE_ENABLE_TRACING" = "true" ]; then
   EXTERNAL_JARS=$(</external_jars/.classpath.txt)
   [ -n "$EXTRA_LIBS" ] && [ -n "${EXTERNAL_JARS}" ] && EXTRA_LIBS=${EXTRA_LIBS}:
-  EXTRA_LIBS="--ext "${EXTRA_LIBS}${EXTERNAL_JARS}
+  EXTRA_LIBS="${EXTRA_LIBS}${EXTERNAL_JARS}"
   echo "Tracing is enabled"
   echo "Classpath will be enriched with these external jars : " ${EXTRA_LIBS}
   if [ -n "$SE_OTEL_SERVICE_NAME" ]; then
@@ -128,6 +135,19 @@ else
   echo "Tracing is disabled"
 fi
 
+if [ "${SE_SESSIONS_MAP_EXTERNAL_DATASTORE}" = "true" ]; then
+  EXTERNAL_JARS=$(</external_jars/.classpath_session_map.txt)
+  [ -n "$EXTRA_LIBS" ] && [ -n "${EXTERNAL_JARS}" ] && EXTRA_LIBS=${EXTRA_LIBS}:
+  EXTRA_LIBS="${EXTRA_LIBS}${EXTERNAL_JARS}"
+fi
+
+if [ -n "${EXTRA_LIBS}" ]; then
+  EXTRA_LIBS="--ext ${EXTRA_LIBS}"
+fi
+
+cat "$CONFIG_FILE"
+echo "Starting Selenium Grid Sessions..."
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   -jar /opt/selenium/selenium-server.jar \
   ${EXTRA_LIBS} sessions \

charts/selenium-grid/CONFIGURATION.md

@@ -18,6 +18,8 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 
 | Repository | Name | Version |
 |------------|------|---------|
+| https://charts.bitnami.com/bitnami | postgresql | 16.2.3 |
+| https://charts.bitnami.com/bitnami | redis | 20.4.0 |
 | https://jaegertracing.github.io/helm-charts | jaeger | 3.3.3 |
 | https://kedacore.github.io/charts | keda | 2.16.0 |
 | https://kubernetes.github.io/ingress-nginx | ingress-nginx | 4.11.3 |
@@ -109,6 +111,8 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | ingress.tls | list | `[]` | TLS backend configuration for ingress resource |
 | busConfigMap.nameOverride | string | `nil` | Override the name of the bus configMap |
 | busConfigMap.annotations | object | `{}` | Custom annotations for configmap |
+| sessionMapConfigMap.nameOverride | string | `nil` | Override the name of the session map configMap |
+| sessionMapConfigMap.annotations | object | `{}` | Custom annotations for configmap |
 | distributorConfigMap.nameOverride | string | `nil` | Override the name of the distributor configMap |
 | distributorConfigMap.defaultMode | int | `493` | Default mode for ConfigMap is mounted as file |
 | distributorConfigMap.extraScriptsImportFrom | string | `"configs/distributor/**"` | Directory where the extra scripts are imported to ConfigMap by default (if given a relative path, it should be in chart's directory) |
@@ -244,6 +248,10 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | components.sessionMap.tolerations | list | `[]` | Tolerations for Session Map pods |
 | components.sessionMap.nodeSelector | object | `{}` | Node selector for Session Map pods |
 | components.sessionMap.priorityClassName | string | `""` | Priority class name for Session Map pods |
+| components.sessionMap.externalDatastore.enabled | bool | `false` | Enable external datastore for Session Map |
+| components.sessionMap.externalDatastore.backend | string | `"postgresql"` | Backend for external datastore (supported: postgresql, redis). Details for each backend are described below config key |
+| components.sessionMap.externalDatastore.postgresql | object | `{"implementation":"org.openqa.selenium.grid.sessionmap.jdbc.JdbcBackedSessionMap","jdbcPassword":"seluser","jdbcUrl":"jdbc:postgresql://{{ $.Release.Name }}-postgresql:5432/selenium_sessions","jdbcUser":"seluser"}` | Configure database backed Session Map (https://www.selenium.dev/documentation/grid/advanced_features/external_datastore/#database-backed-session-map) |
+| components.sessionMap.externalDatastore.redis | object | `{"hostname":"{{ $.Release.Name }}-redis-master","implementation":"org.openqa.selenium.grid.sessionmap.redis.RedisBackedSessionMap","port":"6379","scheme":"redis"}` | Configure Redis backed Session Map (https://www.selenium.dev/documentation/grid/advanced_features/external_datastore/#redis-backed-session-map) |
 | components.sessionQueue.imageRegistry | string | `nil` | Registry to pull the image (this overwrites global.seleniumGrid.imageRegistry parameter) |
 | components.sessionQueue.imageName | string | `"session-queue"` | Session Queue image name |
 | components.sessionQueue.imageTag | string | `nil` | Session Queue image tag (this overwrites global.seleniumGrid.imageTag parameter) |
@@ -590,4 +598,10 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | ingress-nginx | object | `{"controller":{"admissionWebhooks":{"enabled":false}}}` | Configuration for dependency chart ingress-nginx |
 | kube-prometheus-stack | object | `{"cleanPrometheusOperatorObjectNames":true,"prometheus":{"prometheusSpec":{"additionalConfig":{"additionalScrapeConfigs":{"key":"{{ template \"seleniumGrid.monitoring.scrape.key\" $ }}","name":"{{ template \"seleniumGrid.monitoring.exporter.fullname\" $ }}"}}}}}` | Configuration for dependency chart kube-prometheus-stack |
 | jaeger | object | `{"agent":{"enabled":false},"allInOne":{"enabled":true,"extraEnv":[{"name":"QUERY_BASE_PATH","value":"/jaeger"}]},"collector":{"enabled":false},"provisionDataStore":{"cassandra":false},"query":{"enabled":false},"storage":{"type":"badger"}}` | Configuration for dependency chart jaeger |
+| postgresql.enabled | bool | `false` | Enable to install PostgreSQL along with Grid |
+| postgresql.auth | object | `{"database":"selenium_sessions","password":"seluser","username":"seluser"}` | Authentication should be aligned with config in session map |
+| postgresql.primary.initdb.scripts | object | `{"init.sql":"CREATE TABLE IF NOT EXISTS sessions_map(\n  session_ids varchar(256),\n  session_caps text,\n  session_uri varchar(256),\n  session_stereotype text,\n  session_start varchar(256)\n);\n"}` | Initdb scripts for PostgreSQL to create sessions_map table |
+| redis.enabled | bool | `false` | Enable to install Redis along with Grid |
+| redis.architecture | string | `"standalone"` | Setup architecture |
+| redis.auth.enabled | bool | `false` | Disable authentication due to implementation still not supporting it |
 

charts/selenium-grid/Chart.yaml

@@ -22,6 +22,14 @@ dependencies:
   version: 66.3.0
   name: kube-prometheus-stack
   condition: monitoring.enabled, prometheus-stack.enabled
+- repository: https://charts.bitnami.com/bitnami
+  version: 16.2.3
+  name: postgresql
+  condition: postgresql.enabled
+- repository: https://charts.bitnami.com/bitnami
+  version: 20.4.0
+  name: redis
+  condition: redis.enabled
 maintainers:
   - name: SeleniumHQ
     email: selenium-developers@googlegroups.com

charts/selenium-grid/README.md

@@ -40,6 +40,7 @@ This chart enables the creation of a Selenium Grid Server in Kubernetes.
       * [TLS termination in the ingress controller, HTTP/2, and related troubleshooting](#tls-termination-in-the-ingress-controller-http2-and-related-troubleshooting)
     * [Node Registration](#node-registration)
     * [Configuration of tracing observability](#configuration-of-tracing-observability)
+    * [Configuration of Session Map using External Datastore](#configuration-of-session-map-using-external-datastore)
     * [Configuration of Selenium Grid chart](#configuration-of-selenium-grid-chart)
     * [Configuration of KEDA](#configuration-of-keda)
     * [Configuration of Ingress NGINX Controller](#configuration-of-ingress-nginx-controller)
@@ -873,6 +874,38 @@ tracing:
 By default, the exporter is set to `otlp`. It is wide compatibility with many tracing backends.
 Read more: [vendors](https://opentelemetry.io/ecosystem/vendors/) native support OpenTelemetry and guidelines on [integration](https://opentelemetry.io/ecosystem/integrations/)
 
+### Configuration of Session Map using External Datastore
+
+Feature [documentation](https://www.selenium.dev/documentation/grid/advanced_features/external_datastore/). It requires the Grid deployed in distributed mode. The feature is disabled by default.
+
+To enable it with Database backed Session Map, also install PostgreSQL service in the same namespace as Selenium Grid. You can set the following values:
+
+```yaml
+isolateComponents: true
+components:
+  sessionMap:
+    externalDatastore:
+      enabled: true
+      backend: postgresql
+
+postgresql:
+  enabled: true
+```
+
+To enable it with Redis backed Session Map, also install Redis service in the same namespace as Selenium Grid. You can set the following values:
+
+```yaml
+isolateComponents: true
+components:
+  sessionMap:
+    externalDatastore:
+      enabled: true
+      backend: redis
+
+redis:
+  enabled: true
+```
+
 ### Configuration of Selenium Grid chart
 
 Check out list [configuration](https://github.com/SeleniumHQ/docker-selenium/blob/trunk/charts/selenium-grid/CONFIGURATION.md) parameter of chart and their default values for more details.

charts/selenium-grid/templates/_nameHelpers.tpl

@@ -91,6 +91,13 @@ Event bus ConfigMap fullname
 {{- tpl (default (include "seleniumGrid.component.name" (list "selenium-event-bus-config" $)) .Values.busConfigMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{/*
+Session Map ConfigMap fullname
+*/}}
+{{- define "seleniumGrid.sessionMap.configmap.fullname" -}}
+{{- tpl (default (include "seleniumGrid.component.name" (list "selenium-session-map-config" $)) .Values.sessionMapConfigMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
 {{/*
 Router fullname
 */}}

charts/selenium-grid/templates/session-map-configmap.yaml

@@ -0,0 +1,45 @@
+{{- if .Values.isolateComponents }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "seleniumGrid.sessionMap.configmap.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+{{- with .Values.sessionMapConfigMap.annotations }}
+  annotations: {{- toYaml . | nindent 4 }}
+{{- end }}
+  labels:
+    {{- include "seleniumGrid.commonLabels" . | nindent 4 }}
+    {{- with .Values.customLabels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+data:
+{{- if .Values.components.sessionMap.externalDatastore.enabled }}
+  SE_SESSIONS_MAP_EXTERNAL_DATASTORE: "true"
+{{- $backend := .Values.components.sessionMap.externalDatastore.backend -}}
+{{- with (get .Values.components.sessionMap.externalDatastore $backend) }}
+{{- with .scheme }}
+  SE_SESSIONS_MAP_EXTERNAL_SCHEME: {{ tpl . $ | quote }}
+{{- end }}
+{{- with .implementation }}
+  SE_SESSIONS_MAP_EXTERNAL_IMPLEMENTATION: {{ tpl . $ | quote }}
+{{- end }}
+{{- with .hostname }}
+  SE_SESSIONS_MAP_EXTERNAL_HOSTNAME: {{ tpl . $ | quote }}
+{{- end }}
+{{- with .port }}
+  SE_SESSIONS_MAP_EXTERNAL_PORT: {{ tpl . $ | quote }}
+{{- end }}
+{{- with .jdbcUrl }}
+  SE_SESSIONS_MAP_EXTERNAL_JDBC_URL: {{ tpl . $ | quote }}
+{{- end }}
+{{- with .jdbcUser }}
+  SE_SESSIONS_MAP_EXTERNAL_JDBC_USER: {{ tpl . $ | quote }}
+{{- end }}
+{{- with .jdbcPassword }}
+  SE_SESSIONS_MAP_EXTERNAL_JDBC_PASSWORD: {{ tpl . $ | quote }}
+{{- end }}
+{{- end }}
+{{- else }}
+  SE_SESSIONS_MAP_EXTERNAL_DATASTORE: "false"
+{{- end }}
+{{- end }}