From 88a8dd8ba6ddd5bf552dfafa0853486c80c5ca2e Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Tue, 27 May 2025 08:50:05 +0800
Subject: [PATCH 01/19] Add additional kind tests

---
 .github/workflows/build-kind.yaml   | 98 +++++++++++++++++++++++++++++
 Makefile                            | 75 +++++++++++++++++++++-
 scripts/kind/coherence-cluster.yaml | 26 ++++++++
 scripts/kind/kind-config.yaml       | 10 +++
 scripts/kind/kind-label-node.sh     | 26 ++++++++
 scripts/kind/kind.sh                | 13 ++++
 6 files changed, 247 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/build-kind.yaml
 create mode 100644 scripts/kind/coherence-cluster.yaml
 create mode 100644 scripts/kind/kind-config.yaml
 create mode 100755 scripts/kind/kind-label-node.sh
 create mode 100755 scripts/kind/kind.sh

diff --git a/.github/workflows/build-kind.yaml b/.github/workflows/build-kind.yaml
new file mode 100644
index 0000000..f08551a
--- /dev/null
+++ b/.github/workflows/build-kind.yaml
@@ -0,0 +1,98 @@
+# Copyright 2025 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# https://oss.oracle.com/licenses/upl.
+
+# ---------------------------------------------------------------------------
+# Coherence Go Client GitHub Actions - Kind Tests
+# ---------------------------------------------------------------------------
+name: CI Perf Tests
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    strategy:
+      fail-fast: false
+      matrix:
+        go-version:
+          - 1.23.x
+          - 1.24.x
+
+
+#   Checkout the source, we need a depth of zero to fetch all of the history otherwise
+#   the copyright check cannot work out the date of the files from Git.
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+      #   This step will free up disc space on the runner by removing
+      #   lots of things that we do not need.
+    - name: disc
+      shell: bash
+      run: |
+        echo "Listing 100 largest packages"
+        dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100
+        df -h
+        echo "Removing large packages"
+        sudo docker image prune --all --force
+        sudo docker builder prune -a
+        sudo apt-get remove -y '^dotnet-.*' || true
+        sudo apt-get remove -y '^llvm-.*' || true
+        sudo apt-get remove -y 'monodoc-http' || true
+        sudo apt-get remove -y 'php.*' || true
+        sudo apt-get remove -y azure-cli google-cloud-sdk hhvm google-chrome-stable firefox powershell mono-devel || true
+        sudo apt-get autoremove -y || true
+        sudo apt-get clean
+        df -h
+        echo "Removing large directories"
+        rm -rf /usr/share/dotnet/
+        sudo rm -rf /usr/local/lib/android
+        sudo rm -rf /opt/ghc
+        sudo rm -rf /opt/hostedtoolcache/CodeQL
+        df -h
+
+    - name: Cache Go Modules
+      uses: actions/cache@v4
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-mods-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-mods-
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: '${{ matrix.go-version }}'
+
+    - name: Create Kind Cluster
+      shell: bash
+      run: |
+        make kind
+        make create-namespace deploy-operator
+        make deploy-coherence
+
+    - name: Run Tests
+      shell: bash
+      run: |
+        kubectl get pods -n coherence-perf
+
+    - name: Shutdown
+      shell: bash
+      run: |
+        make kind-stop
+
+    - uses: actions/upload-artifact@v4
+      if: failure()
+      with:
+        name: test-output-${{ matrix.go-version }}-${{ matrix.coherenceVersion }}
+        path: build/_output/test-logs
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: test-result-${{ matrix.go-version }}-${{ matrix.coherenceVersion }}
+        path:  test/e2e/perf/results.txt
diff --git a/Makefile b/Makefile
index 102370a..12d6998 100644
--- a/Makefile
+++ b/Makefile
@@ -277,6 +277,80 @@ show-docs:   ## Show the Documentation
 trivy-scan: gettrivy ## Scan the CLI using trivy
 	$(TOOLS_BIN)/trivy fs --cache-dir ${TRIVY_CACHE} --exit-code 1 --skip-dirs "./java" .
 
+# ======================================================================================================================
+# Targets related to running KinD clusters for testing
+# ======================================================================================================================
+##@ KinD
+
+KIND_CLUSTER   ?= go-client
+KIND_IMAGE     ?= "kindest/node:v1.33.0@sha256:91e9ed777db80279c22d1d1068c091b899b2078506e4a0f797fbf6e397c0b0b2"
+KIND_SCRIPTS   := ./scripts/kind
+NAMESPACE      ?= coherence-perf
+OPERATOR_VERSION ?= v3.5.0
+COHERENCE_IMAGE ?= ghcr.io/oracle/coherence-ce:14.1.2-0-2-java17
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Start a Kind cluster
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: kind
+kind:   ## Run a default KinD cluster
+	kind create cluster --name $(KIND_CLUSTER) --wait 10m --config $(KIND_SCRIPTS)/kind-config.yaml --image $(KIND_IMAGE)
+	$(KIND_SCRIPTS)/kind-label-node.sh
+	docker pull $(COHERENCE_IMAGE)
+	kind --name $(KIND_CLUSTER) load docker-image $(COHERENCE_IMAGE)
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Stop and delete the Kind cluster
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: kind-stop
+kind-stop:   ## Stop and delete the KinD cluster
+	kind delete cluster --name $(KIND_CLUSTER)
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Deploy Coherence Operator
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: deploy-operator
+deploy-operator:   ## Deploy the Coherence Operator
+	kubectl apply -f https://github.com/oracle/coherence-operator/releases/download/$(OPERATOR_VERSION)/coherence-operator.yaml
+	kubectl -n coherence wait --timeout=300s --for condition=available deployment/coherence-operator-controller-manager
+
+# ----------------------------------------------------------------------------------------------------------------------
+# UnDeploy Coherence Operator
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: undeploy-operator
+undeploy-operator:   ## UnDeploy the Coherence Operator
+	kubectl delete -f https://github.com/oracle/coherence-operator/releases/download/$(OPERATOR_VERSION)/coherence-operator.yaml || true
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Deploy Coherence Cluster
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: deploy-coherence
+deploy-coherence:   ## Deploy the Coherence Cluster
+	kubectl apply -n $(NAMESPACE) -f $(KIND_SCRIPTS)/coherence-cluster.yaml
+	kubectl -n $(NAMESPACE) wait --timeout=300s --for condition=Ready coherence/perf-cluster
+
+# ----------------------------------------------------------------------------------------------------------------------
+# UnDeploy Coherence Cluster
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: undeploy-coherence
+undeploy-coherence:   ## UnDeploy the Coherence Cluster
+	kubectl delete -n $(NAMESPACE) -f $(KIND_SCRIPTS)/coherence-cluster.yaml || true
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Create Perf namespace
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: create-namespace
+create-namespace:   ## Create the perf test namespace
+	kubectl create namespace $(NAMESPACE)
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Delete Perf namespace
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: delete-namespace
+delete-namespace:   ## Create the perf test namespace
+	kubectl delete namespace $(NAMESPACE) || true
+
 
 # ======================================================================================================================
 # Test targets
@@ -344,7 +418,6 @@ test-v1-base: test-clean test gotestsum $(BUILD_PROPS) ## Run e2e tests with Coh
 test-examples: test-clean gotestsum $(BUILD_PROPS) ## Run examples tests with Coherence
 	./scripts/run-test-examples.sh
 
-
 # ----------------------------------------------------------------------------------------------------------------------
 # Startup cluster members via docker compose
 # ----------------------------------------------------------------------------------------------------------------------
diff --git a/scripts/kind/coherence-cluster.yaml b/scripts/kind/coherence-cluster.yaml
new file mode 100644
index 0000000..aea8dd8
--- /dev/null
+++ b/scripts/kind/coherence-cluster.yaml
@@ -0,0 +1,26 @@
+# Copyright 2025 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# https://oss.oracle.com/licenses/upl.
+apiVersion: coherence.oracle.com/v1
+kind: Coherence
+metadata:
+  name: perf-cluster
+spec:
+  jvm:
+    memory:
+      initialHeapSize: 1g
+      maxHeapSize: 1g
+  replicas: 3
+  readinessProbe:
+    initialDelaySeconds: 10
+    periodSeconds: 10
+  suspendServicesOnShutdown: false
+  image: "ghcr.io/oracle/coherence-ce:14.1.2-0-1-java17"
+  imagePullPolicy: IfNotPresent
+  coherence:
+    management:
+      enabled: true
+  ports:
+    - name: management
+    - name: grpc
+      port: 1408
\ No newline at end of file
diff --git a/scripts/kind/kind-config.yaml b/scripts/kind/kind-config.yaml
new file mode 100644
index 0000000..65fe126
--- /dev/null
+++ b/scripts/kind/kind-config.yaml
@@ -0,0 +1,10 @@
+# Copyright 2025 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# https://oss.oracle.com/licenses/upl.
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+  - role: worker
+  - role: worker
+  - role: worker
diff --git a/scripts/kind/kind-label-node.sh b/scripts/kind/kind-label-node.sh
new file mode 100755
index 0000000..30a12f5
--- /dev/null
+++ b/scripts/kind/kind-label-node.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+#
+# Copyright (c) 2020, 2025, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+
+PREFIX=go-client
+
+kubectl label node ${PREFIX}-worker topology.kubernetes.io/zone=zone-one --overwrite
+kubectl label node ${PREFIX}-worker topology.kubernetes.io/region=one --overwrite
+kubectl label node ${PREFIX}-worker oci.oraclecloud.com/fault-domain=fd-one --overwrite
+kubectl label node ${PREFIX}-worker coherence.oracle.com/site=site-one --overwrite
+kubectl label node ${PREFIX}-worker coherence.oracle.com/rack=rack-one --overwrite
+kubectl label node ${PREFIX}-worker2 topology.kubernetes.io/zone=zone-two --overwrite || true
+kubectl label node ${PREFIX}-worker2 topology.kubernetes.io/region=two --overwrite || true
+kubectl label node ${PREFIX}-worker2 oci.oraclecloud.com/fault-domain=fd-two --overwrite || true
+kubectl label node ${PREFIX}-worker2 coherence.oracle.com/site=site-two --overwrite || true
+kubectl label node ${PREFIX}-worker2 coherence.oracle.com/rack=rack-two --overwrite || true
+kubectl label node ${PREFIX}-worker3 topology.kubernetes.io/zone=zone-three --overwrite || true
+kubectl label node ${PREFIX}-worker3 topology.kubernetes.io/region=three --overwrite || true
+kubectl label node ${PREFIX}-worker3 oci.oraclecloud.com/fault-domain=fd-three --overwrite || true
+kubectl label node ${PREFIX}-worker3 coherence.oracle.com/site=site-three --overwrite || true
+kubectl label node ${PREFIX}-worker3 coherence.oracle.com/rack=rack-three --overwrite || true
+
diff --git a/scripts/kind/kind.sh b/scripts/kind/kind.sh
new file mode 100755
index 0000000..b512202
--- /dev/null
+++ b/scripts/kind/kind.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+#
+# Copyright (c) 2020, 2025, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+
+set -o errexit
+
+# desired cluster name; default is "kind"
+KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-coherence-go-client}"
+
+kind create cluster --name "${KIND_CLUSTER_NAME}" $@

From 40d55c992f3f311d2a8d528903b64b89ef99747d Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Tue, 27 May 2025 08:51:23 +0800
Subject: [PATCH 02/19] Add additional kind tests

---
 .github/workflows/build-kind.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-kind.yaml b/.github/workflows/build-kind.yaml
index f08551a..6e32fb0 100644
--- a/.github/workflows/build-kind.yaml
+++ b/.github/workflows/build-kind.yaml
@@ -5,7 +5,7 @@
 # ---------------------------------------------------------------------------
 # Coherence Go Client GitHub Actions - Kind Tests
 # ---------------------------------------------------------------------------
-name: CI Perf Tests
+name: CI Kind Tests
 
 on:
   workflow_dispatch:

From e0c628ecad77323952b98fbb23b930528cf0e5a8 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Tue, 27 May 2025 08:57:57 +0800
Subject: [PATCH 03/19] Add additional kind tests

---
 .github/workflows/build-kind.yaml | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-kind.yaml b/.github/workflows/build-kind.yaml
index 6e32fb0..cdfdaf7 100644
--- a/.github/workflows/build-kind.yaml
+++ b/.github/workflows/build-kind.yaml
@@ -8,7 +8,15 @@
 name: CI Kind Tests
 
 on:
+  push:
+    branches:
+      - '*'
   workflow_dispatch:
+    inputs:
+      go-version:
+        description: "Go version (comma-separated for matrix)"
+        required: false
+        default: "1.23.x,1.24.x"
 
 jobs:
   build:
@@ -22,7 +30,6 @@ jobs:
           - 1.23.x
           - 1.24.x
 
-
 #   Checkout the source, we need a depth of zero to fetch all of the history otherwise
 #   the copyright check cannot work out the date of the files from Git.
     steps:
@@ -30,6 +37,13 @@ jobs:
       with:
         fetch-depth: 0
 
+    - name: Skip if Pull Request
+      run: |
+        if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+          echo "This is a PR"
+          exit 0
+        fi
+
       #   This step will free up disc space on the runner by removing
       #   lots of things that we do not need.
     - name: disc

From 76c4df08d6c07164481746ff3e4cf45daf69c2b9 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Tue, 27 May 2025 09:03:29 +0800
Subject: [PATCH 04/19] Fix workflow

---
 .github/workflows/build-kind.yaml | 13 ++++++-------
 Makefile                          |  2 --
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/build-kind.yaml b/.github/workflows/build-kind.yaml
index cdfdaf7..9036054 100644
--- a/.github/workflows/build-kind.yaml
+++ b/.github/workflows/build-kind.yaml
@@ -37,16 +37,10 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Skip if Pull Request
-      run: |
-        if [[ "${{ github.event_name }}" == "pull_request" ]]; then
-          echo "This is a PR"
-          exit 0
-        fi
-
       #   This step will free up disc space on the runner by removing
       #   lots of things that we do not need.
     - name: disc
+      if: github.event_name != 'pull_request'
       shell: bash
       run: |
         echo "Listing 100 largest packages"
@@ -71,6 +65,7 @@ jobs:
         df -h
 
     - name: Cache Go Modules
+      if: github.event_name != 'pull_request'
       uses: actions/cache@v4
       with:
         path: ~/go/pkg/mod
@@ -79,11 +74,13 @@ jobs:
           ${{ runner.os }}-go-mods-
 
     - name: Set up Go
+      if: github.event_name != 'pull_request'
       uses: actions/setup-go@v5
       with:
         go-version: '${{ matrix.go-version }}'
 
     - name: Create Kind Cluster
+      if: github.event_name != 'pull_request'
       shell: bash
       run: |
         make kind
@@ -91,11 +88,13 @@ jobs:
         make deploy-coherence
 
     - name: Run Tests
+      if: github.event_name != 'pull_request'
       shell: bash
       run: |
         kubectl get pods -n coherence-perf
 
     - name: Shutdown
+      if: github.event_name != 'pull_request'
       shell: bash
       run: |
         make kind-stop
diff --git a/Makefile b/Makefile
index 12d6998..4dd6c14 100644
--- a/Makefile
+++ b/Makefile
@@ -296,8 +296,6 @@ COHERENCE_IMAGE ?= ghcr.io/oracle/coherence-ce:14.1.2-0-2-java17
 kind:   ## Run a default KinD cluster
 	kind create cluster --name $(KIND_CLUSTER) --wait 10m --config $(KIND_SCRIPTS)/kind-config.yaml --image $(KIND_IMAGE)
 	$(KIND_SCRIPTS)/kind-label-node.sh
-	docker pull $(COHERENCE_IMAGE)
-	kind --name $(KIND_CLUSTER) load docker-image $(COHERENCE_IMAGE)
 
 # ----------------------------------------------------------------------------------------------------------------------
 # Stop and delete the Kind cluster

From cfa099e60239cc63661af154f5e341711f540cf3 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Tue, 27 May 2025 09:10:26 +0800
Subject: [PATCH 05/19] Fix workflow

---
 .github/workflows/build-kind.yaml | 37 +++++--------------------------
 1 file changed, 5 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/build-kind.yaml b/.github/workflows/build-kind.yaml
index 9036054..ce212ff 100644
--- a/.github/workflows/build-kind.yaml
+++ b/.github/workflows/build-kind.yaml
@@ -37,35 +37,8 @@ jobs:
       with:
         fetch-depth: 0
 
-      #   This step will free up disc space on the runner by removing
-      #   lots of things that we do not need.
-    - name: disc
-      if: github.event_name != 'pull_request'
-      shell: bash
-      run: |
-        echo "Listing 100 largest packages"
-        dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100
-        df -h
-        echo "Removing large packages"
-        sudo docker image prune --all --force
-        sudo docker builder prune -a
-        sudo apt-get remove -y '^dotnet-.*' || true
-        sudo apt-get remove -y '^llvm-.*' || true
-        sudo apt-get remove -y 'monodoc-http' || true
-        sudo apt-get remove -y 'php.*' || true
-        sudo apt-get remove -y azure-cli google-cloud-sdk hhvm google-chrome-stable firefox powershell mono-devel || true
-        sudo apt-get autoremove -y || true
-        sudo apt-get clean
-        df -h
-        echo "Removing large directories"
-        rm -rf /usr/share/dotnet/
-        sudo rm -rf /usr/local/lib/android
-        sudo rm -rf /opt/ghc
-        sudo rm -rf /opt/hostedtoolcache/CodeQL
-        df -h
-
     - name: Cache Go Modules
-      if: github.event_name != 'pull_request'
+      if: github.event_name == 'workflow_dispatch'
       uses: actions/cache@v4
       with:
         path: ~/go/pkg/mod
@@ -74,13 +47,13 @@ jobs:
           ${{ runner.os }}-go-mods-
 
     - name: Set up Go
-      if: github.event_name != 'pull_request'
+      if: github.event_name == 'workflow_dispatch'
       uses: actions/setup-go@v5
       with:
         go-version: '${{ matrix.go-version }}'
 
     - name: Create Kind Cluster
-      if: github.event_name != 'pull_request'
+      if: github.event_name == 'workflow_dispatch'
       shell: bash
       run: |
         make kind
@@ -88,13 +61,13 @@ jobs:
         make deploy-coherence
 
     - name: Run Tests
-      if: github.event_name != 'pull_request'
+      if: github.event_name == 'workflow_dispatch'
       shell: bash
       run: |
         kubectl get pods -n coherence-perf
 
     - name: Shutdown
-      if: github.event_name != 'pull_request'
+      if: github.event_name == 'workflow_dispatch'
       shell: bash
       run: |
         make kind-stop

From 853ed2562f0174b939ba88e2bedecc281db06f70 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Tue, 27 May 2025 13:45:04 +0800
Subject: [PATCH 06/19] More test updates

---
 .github/workflows/build-kind.yaml   |  43 +++--
 .gitignore                          |   1 +
 Makefile                            |  50 +++++-
 coherence/common.go                 |  72 +++++++-
 coherence/session.go                |   4 +
 scripts/kind/coherence-cluster.yaml |   4 +-
 scripts/kind/load-schools.yaml      |  32 ++++
 scripts/kind/roll-cluster.sh        |  42 +++++
 scripts/kind/test-schools.yaml      |  35 ++++
 test/e2e/kind/Dockerfile            |  14 ++
 test/e2e/kind/main.go               | 267 ++++++++++++++++++++++++++++
 11 files changed, 538 insertions(+), 26 deletions(-)
 create mode 100644 scripts/kind/load-schools.yaml
 create mode 100755 scripts/kind/roll-cluster.sh
 create mode 100644 scripts/kind/test-schools.yaml
 create mode 100644 test/e2e/kind/Dockerfile
 create mode 100644 test/e2e/kind/main.go

diff --git a/.github/workflows/build-kind.yaml b/.github/workflows/build-kind.yaml
index ce212ff..2ad7ef8 100644
--- a/.github/workflows/build-kind.yaml
+++ b/.github/workflows/build-kind.yaml
@@ -27,7 +27,6 @@ jobs:
       fail-fast: false
       matrix:
         go-version:
-          - 1.23.x
           - 1.24.x
 
 #   Checkout the source, we need a depth of zero to fetch all of the history otherwise
@@ -38,7 +37,7 @@ jobs:
         fetch-depth: 0
 
     - name: Cache Go Modules
-      if: github.event_name == 'workflow_dispatch'
+    #  if: github.event_name == 'workflow_dispatch'
       uses: actions/cache@v4
       with:
         path: ~/go/pkg/mod
@@ -47,38 +46,56 @@ jobs:
           ${{ runner.os }}-go-mods-
 
     - name: Set up Go
-      if: github.event_name == 'workflow_dispatch'
+  #    if: github.event_name == 'workflow_dispatch'
       uses: actions/setup-go@v5
       with:
         go-version: '${{ matrix.go-version }}'
 
     - name: Create Kind Cluster
-      if: github.event_name == 'workflow_dispatch'
+ #     if: github.event_name == 'workflow_dispatch'
       shell: bash
       run: |
         make kind
         make create-namespace deploy-operator
         make deploy-coherence
+        make build-go-client
+        make load-schools
+        mkdir -p build/_output/
 
     - name: Run Tests
-      if: github.event_name == 'workflow_dispatch'
+   #   if: github.event_name == 'workflow_dispatch'
       shell: bash
       run: |
-        kubectl get pods -n coherence-perf
+        NAMESPACE=coherence-perf
+        # Put the tail into the background
+        POD=$(kubectl get pods -n $NAMESPACE | grep perf-go-client | awk '{print $1}') 
+        kubectl logs $POD -n $NAMESPACE -f > build/_output/test-logs/perf-go-client.log 2>&1 &
+        TAIL_PID=$!
+        
+        # port forward the http port
+        kubectl port-forward -n $NAMESPACE pod/$POD 8080:8080 > build/_output/test-logs/port-forward.log 2>&1 &
+        PORT_FORWARD_PID=$!
+        
+        # run curl requests
+        while : ; do curl -s -v http://localhost:8080/api/schools?q=[1-2] | jq length; sleep 0.5; done > build/_output/test-logs/curl.log 2>&1 &
+        CURL_PID=$!
+        
+        # Start the rolling restart
+        ./scripts/kind/roll-cluster.sh | tee build/_output/test-logs/rolling-restart.log
+        
+        make undeploy-test-schools
+        
+        kill -9 $CURL_PID && sleep 5 && kill -9 $PORT_FORWARD_PID && kill -9 $TAIL_PID
+        
 
     - name: Shutdown
-      if: github.event_name == 'workflow_dispatch'
+ #     if: github.event_name == 'workflow_dispatch'
       shell: bash
       run: |
         make kind-stop
 
     - uses: actions/upload-artifact@v4
-      if: failure()
       with:
-        name: test-output-${{ matrix.go-version }}-${{ matrix.coherenceVersion }}
+        name: test-output-${{ matrix.go-version }}-test-logs
         path: build/_output/test-logs
 
-    - uses: actions/upload-artifact@v4
-      with:
-        name: test-result-${{ matrix.go-version }}-${{ matrix.coherenceVersion }}
-        path:  test/e2e/perf/results.txt
diff --git a/.gitignore b/.gitignore
index 42b03a9..473be54 100644
--- a/.gitignore
+++ b/.gitignore
@@ -41,3 +41,4 @@ certs/
 release/
 etc/
 coherence/coherence.test
+runner
diff --git a/Makefile b/Makefile
index 4dd6c14..443be02 100644
--- a/Makefile
+++ b/Makefile
@@ -178,6 +178,7 @@ copyright: getcopyright ## Check copyright headers
 	  -X proto/ \
 	  -X /Dockerfile \
 	  -X .Dockerfile \
+	  -X runner \
 	  -X go.sum \
 	  -X HEADER.txt \
 	  -X .iml \
@@ -282,12 +283,15 @@ trivy-scan: gettrivy ## Scan the CLI using trivy
 # ======================================================================================================================
 ##@ KinD
 
-KIND_CLUSTER   ?= go-client
-KIND_IMAGE     ?= "kindest/node:v1.33.0@sha256:91e9ed777db80279c22d1d1068c091b899b2078506e4a0f797fbf6e397c0b0b2"
-KIND_SCRIPTS   := ./scripts/kind
-NAMESPACE      ?= coherence-perf
+KIND_CLUSTER     ?= go-client
+KIND_IMAGE       ?= "kindest/node:v1.33.0@sha256:91e9ed777db80279c22d1d1068c091b899b2078506e4a0f797fbf6e397c0b0b2"
+KIND_SCRIPTS     := ./scripts/kind
+NAMESPACE        ?= coherence-perf
 OPERATOR_VERSION ?= v3.5.0
-COHERENCE_IMAGE ?= ghcr.io/oracle/coherence-ce:14.1.2-0-2-java17
+COHERENCE_IMAGE  ?= ghcr.io/oracle/coherence-ce:14.1.2-0-2-java17
+INITIAL_HEAP     ?= 1g
+GO_CLIENT_ARCH   ?= amd64
+GO_IMAGE         ?= perf-go-client:1.0.0
 
 # ----------------------------------------------------------------------------------------------------------------------
 # Start a Kind cluster
@@ -324,9 +328,41 @@ undeploy-operator:   ## UnDeploy the Coherence Operator
 # ----------------------------------------------------------------------------------------------------------------------
 .PHONY: deploy-coherence
 deploy-coherence:   ## Deploy the Coherence Cluster
-	kubectl apply -n $(NAMESPACE) -f $(KIND_SCRIPTS)/coherence-cluster.yaml
+	envsubst <  $(KIND_SCRIPTS)/coherence-cluster.yaml | kubectl apply -n $(NAMESPACE) -f -
+	sleep 5
 	kubectl -n $(NAMESPACE) wait --timeout=300s --for condition=Ready coherence/perf-cluster
 
+#-----------------------------------------------------------------------------------------------------------------------
+# Deploy Coherence Cluster
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: build-go-client
+build-go-client:   ## Make Go Client
+	cd test/e2e/kind && CGO_ENABLED=0 GOOS=linux GOARCH=$(GO_CLIENT_ARCH) GO111MODULE=on go build -trimpath -o runner . && docker build --no-cache -t $(GO_IMAGE) .
+	kind --name $(KIND_CLUSTER) load docker-image $(GO_IMAGE)
+
+#-----------------------------------------------------------------------------------------------------------------------
+# Load Schools Data
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: load-schools
+load-schools:   ## Load Schools
+	kubectl -n $(NAMESPACE) apply -f $(KIND_SCRIPTS)/load-schools.yaml
+	kubectl wait -n $(NAMESPACE) --timeout=1200s --for condition=Complete job/go-perf-load-schools
+	kubectl -n $(NAMESPACE) delete -f $(KIND_SCRIPTS)/load-schools.yaml || true
+
+#-----------------------------------------------------------------------------------------------------------------------
+# Test Schools
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: deploy-test-schools
+deploy-test-schools:   ## Deploy Test Schools
+	kubectl -n $(NAMESPACE) apply -f $(KIND_SCRIPTS)/test-schools.yaml
+
+#-----------------------------------------------------------------------------------------------------------------------
+# Stop Schools Test
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: undeploy-test-schools
+undeploy-test-schools:   ## Undeploy Test Schools
+	kubectl -n $(NAMESPACE) delete -f $(KIND_SCRIPTS)/test-schools.yaml
+
 # ----------------------------------------------------------------------------------------------------------------------
 # UnDeploy Coherence Cluster
 # ----------------------------------------------------------------------------------------------------------------------
@@ -334,7 +370,6 @@ deploy-coherence:   ## Deploy the Coherence Cluster
 undeploy-coherence:   ## UnDeploy the Coherence Cluster
 	kubectl delete -n $(NAMESPACE) -f $(KIND_SCRIPTS)/coherence-cluster.yaml || true
 
-
 # ----------------------------------------------------------------------------------------------------------------------
 # Create Perf namespace
 # ----------------------------------------------------------------------------------------------------------------------
@@ -349,7 +384,6 @@ create-namespace:   ## Create the perf test namespace
 delete-namespace:   ## Create the perf test namespace
 	kubectl delete namespace $(NAMESPACE) || true
 
-
 # ======================================================================================================================
 # Test targets
 # ======================================================================================================================
diff --git a/coherence/common.go b/coherence/common.go
index 09e86d7..2919a90 100644
--- a/coherence/common.go
+++ b/coherence/common.go
@@ -57,6 +57,8 @@ const (
 	integerMaxValue = 2147483647
 
 	ListenAll InvalidationStrategyType = 0
+
+	panicWarning = "recovered from panic, possible connection closed while received channel data: %v"
 )
 
 var (
@@ -636,6 +638,14 @@ func executeGetAll[K comparable, V any](ctx context.Context, bc *baseClient[K, V
 	}
 
 	go func() {
+		defer func() {
+			// catch panic of closed channel read in rare circumstances
+			if r := recover(); r != nil {
+				logMessage(WARNING, panicWarning, r)
+				return
+			}
+		}()
+
 		if cancel != nil {
 			defer cancel()
 		}
@@ -1185,6 +1195,14 @@ func executeInvokeAllFilterOrKeys[K comparable, V any, R any](ctx context.Contex
 	}
 
 	go func() {
+		defer func() {
+			// catch panic of closed channel read in rare circumstances
+			if r := recover(); r != nil {
+				logMessage(WARNING, panicWarning, r)
+				return
+			}
+		}()
+
 		if cancel != nil {
 			defer cancel()
 		}
@@ -1264,10 +1282,18 @@ func executeKeySet[K comparable, V any](ctx context.Context, bc *baseClient[K, V
 	}
 
 	go func() {
+		defer func() {
+			// catch panic of closed channel read in rare circumstances
+			if r := recover(); r != nil {
+				logMessage(WARNING, panicWarning, r)
+				return
+			}
+		}()
+
 		for {
 			result, err1 := iter.Next()
 
-			if err1 == ErrDone {
+			if errors.Is(err1, ErrDone) {
 				close(ch)
 				return
 			} else if err1 != nil {
@@ -1307,6 +1333,14 @@ func executeKeySetFilter[K comparable, V any](ctx context.Context, bc *baseClien
 	}
 
 	go func() {
+		defer func() {
+			// catch panic of closed channel read in rare circumstances
+			if r := recover(); r != nil {
+				logMessage(WARNING, panicWarning, r)
+				return
+			}
+		}()
+
 		if cancel != nil {
 			defer cancel()
 		}
@@ -1840,10 +1874,18 @@ func executeEntrySet[K comparable, V any](ctx context.Context, bc *baseClient[K,
 	}
 
 	go func() {
+		defer func() {
+			// catch panic of closed channel read in rare circumstances
+			if r := recover(); r != nil {
+				logMessage(WARNING, panicWarning, r)
+				return
+			}
+		}()
+
 		for {
 			result, err1 := iter.Next()
 
-			if err1 == ErrDone {
+			if errors.Is(err1, ErrDone) {
 				close(ch)
 				return
 			} else if err1 != nil {
@@ -1892,6 +1934,14 @@ func executeEntrySetFilter[K comparable, V any, E any](ctx context.Context, bc *
 	}
 
 	go func() {
+		defer func() {
+			// catch panic of closed channel read in rare circumstances
+			if r := recover(); r != nil {
+				logMessage(WARNING, panicWarning, r)
+				return
+			}
+		}()
+
 		if cancel != nil {
 			defer cancel()
 		}
@@ -1986,6 +2036,14 @@ func executeValues[K comparable, V any, E any](ctx context.Context, bc *baseClie
 	}
 
 	go func() {
+		defer func() {
+			// catch panic of closed channel read in rare circumstances
+			if r := recover(); r != nil {
+				logMessage(WARNING, panicWarning, r)
+				return
+			}
+		}()
+
 		if cancel != nil {
 			defer cancel()
 		}
@@ -2056,10 +2114,18 @@ func executeValuesNoFilter[K comparable, V any](ctx context.Context, bc *baseCli
 	}
 
 	go func() {
+		defer func() {
+			// catch panic of closed channel read in rare circumstances
+			if r := recover(); r != nil {
+				logMessage(WARNING, panicWarning, r)
+				return
+			}
+		}()
+
 		for {
 			result, err1 := iter.Next()
 
-			if err1 == ErrDone {
+			if errors.Is(err1, ErrDone) {
 				close(ch)
 				return
 			} else if err1 != nil {
diff --git a/coherence/session.go b/coherence/session.go
index e4bcaca..4b8048e 100644
--- a/coherence/session.go
+++ b/coherence/session.go
@@ -484,6 +484,7 @@ func (s *Session) GetReadyTimeout() time.Duration {
 func (s *Session) ensureConnection() error {
 	s.connectMutex.Lock()
 	defer s.connectMutex.Unlock()
+	logMessage(DEBUG, "ensureConnection()")
 
 	if s.firstConnectAttempted {
 		// We have previously tried to connect so check that the connect state is connected
@@ -664,6 +665,7 @@ func waitForReady(s *Session) error {
 
 	// try to connect up until timeout, then throw err if not available
 	timeout := time.Now().Add(readyTimeout)
+
 	for {
 		if time.Now().After(timeout) {
 			return fmt.Errorf("unable to connect to %s after ready timeout of %v", s.sessOpts.Address, readyTimeout)
@@ -677,6 +679,8 @@ func waitForReady(s *Session) error {
 		if state == connectivity.Ready {
 			return nil
 		}
+		logMessage(DEBUG, "state=%v", state)
+
 		if !messageLogged {
 			logMessage(INFO, "Session [%s] State is %v, waiting until ready timeout of %v for valid connection",
 				s.sessionID, state, readyTimeout)
diff --git a/scripts/kind/coherence-cluster.yaml b/scripts/kind/coherence-cluster.yaml
index aea8dd8..90dd83a 100644
--- a/scripts/kind/coherence-cluster.yaml
+++ b/scripts/kind/coherence-cluster.yaml
@@ -8,14 +8,14 @@ metadata:
 spec:
   jvm:
     memory:
-      initialHeapSize: 1g
+      initialHeapSize: $INITIAL_HEAP
       maxHeapSize: 1g
   replicas: 3
   readinessProbe:
     initialDelaySeconds: 10
     periodSeconds: 10
   suspendServicesOnShutdown: false
-  image: "ghcr.io/oracle/coherence-ce:14.1.2-0-1-java17"
+  image: "ghcr.io/oracle/coherence-ce:14.1.2-0-2-java17"
   imagePullPolicy: IfNotPresent
   coherence:
     management:
diff --git a/scripts/kind/load-schools.yaml b/scripts/kind/load-schools.yaml
new file mode 100644
index 0000000..6849a44
--- /dev/null
+++ b/scripts/kind/load-schools.yaml
@@ -0,0 +1,32 @@
+# Copyright 2025 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# https://oss.oracle.com/licenses/upl.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: go-perf-load-schools
+spec:
+  completions: 1     # total times to run
+  parallelism: 1     # how many pods to run at the same time
+  template:
+    metadata:
+      labels:
+        app: go-client
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: perf-go-client
+          image: "perf-go-client:1.0.0"
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: COHERENCE_SERVER_ADDRESS
+              value: "perf-cluster-grpc:1408"
+            - name: TEST_TYPE
+              value: "load"
+            - name: CACHE_COUNT
+              value: "250000"
+          resources:
+            requests:
+              memory: "1024Mi"
+            limits:
+              memory: "1024Mi"
\ No newline at end of file
diff --git a/scripts/kind/roll-cluster.sh b/scripts/kind/roll-cluster.sh
new file mode 100755
index 0000000..903dff6
--- /dev/null
+++ b/scripts/kind/roll-cluster.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+#
+# Copyright (c) 2020, 2025, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+
+set -o errexit
+maxIterations=100
+
+if [ ! -z "$MAX_ITERATIONS" ]; then
+  maxIterations=$MAX_ITERATIONS
+fi
+
+if [ -z "$NAMESPACE" ]; then
+  NAMESPACE=coherence-perf
+fi
+
+echo "`date`: Starting rolling restart, iterations=$maxIterations"
+
+for i in $(seq 1 $maxIterations); do
+
+  echo "`date`: Iteration $i of $maxIterations"
+
+  if [ $((i%2)) -eq 0 ]; then
+    INITIAL_HEAP=1000m
+  else
+    INITIAL_HEAP=1001m
+  fi
+
+  export INITIAL_HEAP
+
+  envsubst < ./scripts/kind/coherence-cluster.yaml | kubectl -n $NAMESPACE apply -f -
+
+  echo "`date`: Waiting for cluster to be updated..."
+  kubectl rollout status statefulset/perf-cluster -n $NAMESPACE --timeout=300s
+
+  echo "`date`: Sleeping 10..."
+  sleep 10
+done
+
+
diff --git a/scripts/kind/test-schools.yaml b/scripts/kind/test-schools.yaml
new file mode 100644
index 0000000..8036674
--- /dev/null
+++ b/scripts/kind/test-schools.yaml
@@ -0,0 +1,35 @@
+# Copyright 2025 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# https://oss.oracle.com/licenses/upl.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: perf-go-client
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        app: perf-go-client
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: go-client
+          image: "perf-go-client:1.0.0"
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: COHERENCE_SERVER_ADDRESS
+              value: "perf-cluster-grpc:1408"
+            - name: COHERENCE_READY_TIMEOUT
+              value: "15000"
+            - name: TEST_TYPE
+              value: "runTest"
+            - name: COHERENCE_LOG_LEVEL
+              value: "ALL"
+          resources:
+            requests:
+              memory: "512Mi"
+            limits:
+              memory: "512Mi"
+          ports:
+            - containerPort: 8080
\ No newline at end of file
diff --git a/test/e2e/kind/Dockerfile b/test/e2e/kind/Dockerfile
new file mode 100644
index 0000000..0c5cf5c
--- /dev/null
+++ b/test/e2e/kind/Dockerfile
@@ -0,0 +1,14 @@
+# ----------------------------------------------------------------------------------------------------------------------
+# Copyright (c) 2025, Oracle and/or its affiliates.
+#
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+
+FROM scratch
+
+COPY runner /files/runner
+
+EXPOSE 8080
+
+ENTRYPOINT ["/files/runner"]
+CMD ["-h"]
diff --git a/test/e2e/kind/main.go b/test/e2e/kind/main.go
new file mode 100644
index 0000000..eec0ca6
--- /dev/null
+++ b/test/e2e/kind/main.go
@@ -0,0 +1,267 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates.
+ * Licensed under the Universal Permissive License v 1.0 as shown at
+ * https://oss.oracle.com/licenses/upl.
+ */
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/oracle/coherence-go-client/v2/coherence"
+	"github.com/oracle/coherence-go-client/v2/coherence/extractors"
+	"github.com/oracle/coherence-go-client/v2/coherence/filters"
+	"log"
+	"math/rand/v2"
+	"net/http"
+	"os"
+	"strconv"
+	"time"
+)
+
+type School struct {
+	ID               int    `json:"id"`
+	City             string `json:"city"`
+	State            string `json:"state"`
+	MarketSegment    string `json:"marketSegment"`
+	CountryCode      string `json:"countryCode"`
+	SchoolName       string `json:"schoolName"`
+	LastModifiedDate string `json:"lastModifiedDate"` // "2006-01-02 15:04:05"
+}
+
+var (
+	session      *coherence.Session
+	schoolsCache coherence.NamedCache[int, School]
+
+	marketSegmentExtractor = extractors.Extract[string]("marketSegment")
+	marketSegmentFilter    = filters.Equal[string](marketSegmentExtractor, "PRIVATE")
+
+	rnd = rand.New(rand.NewPCG(uint64(time.Now().UnixNano()), uint64(time.Now().UnixNano()))) //nolint:gosec // this is a test
+)
+
+func main() {
+	var (
+		count    = 250_000
+		testType = "runTest"
+		err      error
+	)
+	if err = initializeCoherence(); err != nil {
+		log.Fatalf("failed to initialize coherence: %v", err)
+	}
+
+	countOverride := os.Getenv("CACHE_COUNT")
+	if countOverride != "" {
+		count, _ = strconv.Atoi(countOverride)
+	}
+
+	if value := os.Getenv("TEST_TYPE"); value != "" {
+		testType = value
+	}
+
+	if testType == "load" {
+		err = populateSchoolsCache(count)
+	} else {
+		err = runTest()
+	}
+	if err != nil {
+		log.Fatalf("failed to run %s: %v", testType, err)
+	}
+}
+
+func initializeCoherence() error {
+	var err error
+	session, err = coherence.NewSession(context.Background(), coherence.WithPlainText())
+	if err != nil {
+		return err
+	}
+
+	// add a reconnect listener
+	listener := coherence.NewSessionLifecycleListener().
+		OnDisconnected(func(e coherence.SessionLifecycleEvent) {
+			log.Printf("Session [%s] disconnected\n", e.Type())
+		})
+
+	session.AddSessionLifecycleListener(listener)
+
+	schoolsCache, err = coherence.GetNamedCache[int, School](session, "schools")
+
+	return err
+}
+
+func runTest() error {
+	http.HandleFunc("/api/schools", schoolsHandler)
+
+	fmt.Println("Server running on port 8080")
+	log.Fatal(http.ListenAndServe(":8080", nil)) //nolint:gosec // G404: weak RNG
+
+	return nil
+}
+
+func schoolsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := context.Background()
+
+	switch r.Method {
+
+	case http.MethodGet:
+		var (
+			count int
+			start = time.Now()
+		)
+
+		defer func() {
+			duration := time.Since(start)
+			log.Printf("Retrieved %d entries in %s", count, duration)
+		}()
+
+		var people = make([]School, 0)
+
+		if session.IsClosed() {
+			log.Printf("Session [%s] is closed, calling initializeCoherence()", session.ID())
+			err := initializeCoherence()
+			if err != nil {
+				log.Printf("failed to initialize coherence: %v", err)
+				http.Error(w, "not connected", http.StatusInternalServerError)
+				return
+			}
+		}
+
+		for ch := range schoolsCache.ValuesFilter(ctx, marketSegmentFilter) {
+			if ch.Err != nil {
+				http.Error(w, ch.Err.Error(), http.StatusInternalServerError)
+				return
+			}
+			people = append(people, ch.Value)
+			count++
+		}
+		_ = json.NewEncoder(w).Encode(people)
+		return
+	default:
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func populateSchoolsCache(count int) error {
+	var (
+		buffer         = make(map[int]School)
+		err            error
+		batchSize      = 10_000
+		marketSegments = []string{"K-12", "MAGNET", "CHARTER", "VIRTUAL", "RELIGIOUS", "BOARDING", "PRIVATE"}
+		ctx            = context.Background()
+	)
+
+	if err = schoolsCache.Clear(ctx); err != nil {
+		return err
+	}
+
+	if err = coherence.AddIndex[int, School](ctx, schoolsCache, marketSegmentExtractor, true); err != nil {
+		return err
+	}
+
+	for i := 1; i <= count; i++ {
+		modifiedDate := time.Now().UTC().Format(time.DateTime)
+
+		randomCity := randomUSCity()
+		buffer[i] = School{
+			ID:               i,
+			City:             randomCity.City,
+			State:            randomCity.State,
+			MarketSegment:    randomize(marketSegments),
+			CountryCode:      "US",
+			SchoolName:       fmt.Sprintf("School Name %d", i),
+			LastModifiedDate: modifiedDate,
+		}
+		if i%batchSize == 0 {
+			err = schoolsCache.PutAll(ctx, buffer)
+			if err != nil {
+				return err
+			}
+			buffer = make(map[int]School)
+			log.Println("PutAll", i)
+		}
+	}
+	if len(buffer) > 0 {
+		return schoolsCache.PutAll(ctx, buffer)
+	}
+
+	return nil
+}
+
+func randomize(arr []string) string {
+	if len(arr) == 0 {
+		return ""
+	}
+	return arr[rnd.IntN(len(arr))]
+}
+
+type USCity struct {
+	City  string
+	State string
+}
+
+var cities = []USCity{
+	// California (CA)
+	{"Los Angeles", "CA"}, {"San Diego", "CA"}, {"San Jose", "CA"}, {"San Francisco", "CA"}, {"Fresno", "CA"},
+	{"Sacramento", "CA"}, {"Long Beach", "CA"}, {"Oakland", "CA"}, {"Bakersfield", "CA"}, {"Anaheim", "CA"},
+	{"Santa Ana", "CA"}, {"Riverside", "CA"}, {"Stockton", "CA"}, {"Irvine", "CA"}, {"Chula Vista", "CA"},
+	{"Fremont", "CA"}, {"San Bernardino", "CA"}, {"Modesto", "CA"}, {"Oxnard", "CA"}, {"Fontana", "CA"},
+
+	// Texas (TX)
+	{"Houston", "TX"}, {"San Antonio", "TX"}, {"Dallas", "TX"}, {"Austin", "TX"}, {"Fort Worth", "TX"},
+	{"El Paso", "TX"}, {"Arlington", "TX"}, {"Corpus Christi", "TX"}, {"Plano", "TX"}, {"Laredo", "TX"},
+	{"Lubbock", "TX"}, {"Garland", "TX"}, {"Irving", "TX"}, {"Amarillo", "TX"}, {"Grand Prairie", "TX"},
+	{"McKinney", "TX"}, {"Frisco", "TX"}, {"Brownsville", "TX"}, {"Pasadena", "TX"}, {"Killeen", "TX"},
+
+	// Florida (FL)
+	{"Jacksonville", "FL"}, {"Miami", "FL"}, {"Tampa", "FL"}, {"Orlando", "FL"}, {"St. Petersburg", "FL"},
+	{"Hialeah", "FL"}, {"Tallahassee", "FL"}, {"Fort Lauderdale", "FL"}, {"Port St. Lucie", "FL"}, {"Cape Coral", "FL"},
+	{"Pembroke Pines", "FL"}, {"Hollywood", "FL"}, {"Miramar", "FL"}, {"Gainesville", "FL"}, {"Coral Springs", "FL"},
+	{"Clearwater", "FL"}, {"Palm Bay", "FL"}, {"Pompano Beach", "FL"}, {"West Palm Beach", "FL"}, {"Lakeland", "FL"},
+
+	// New York (NY)
+	{"New York", "NY"}, {"Buffalo", "NY"}, {"Rochester", "NY"}, {"Yonkers", "NY"}, {"Syracuse", "NY"},
+	{"Albany", "NY"}, {"New Rochelle", "NY"}, {"Mount Vernon", "NY"}, {"Schenectady", "NY"}, {"Utica", "NY"},
+	{"White Plains", "NY"}, {"Troy", "NY"}, {"Niagara Falls", "NY"}, {"Binghamton", "NY"}, {"Freeport", "NY"},
+	{"Valley Stream", "NY"}, {"Hempstead", "NY"}, {"Levittown", "NY"}, {"Brentwood", "NY"}, {"Irondequoit", "NY"},
+
+	// Illinois (IL)
+	{"Chicago", "IL"}, {"Aurora", "IL"}, {"Naperville", "IL"}, {"Joliet", "IL"}, {"Rockford", "IL"},
+	{"Springfield", "IL"}, {"Elgin", "IL"}, {"Peoria", "IL"}, {"Champaign", "IL"}, {"Waukegan", "IL"},
+	{"Cicero", "IL"}, {"Bloomington", "IL"}, {"Arlington Heights", "IL"}, {"Evanston", "IL"}, {"Decatur", "IL"},
+	{"Schaumburg", "IL"}, {"Bolingbrook", "IL"}, {"Palatine", "IL"}, {"Skokie", "IL"}, {"Des Plaines", "IL"},
+
+	// Pennsylvania (PA)
+	{"Philadelphia", "PA"}, {"Pittsburgh", "PA"}, {"Allentown", "PA"}, {"Erie", "PA"}, {"Reading", "PA"},
+	{"Scranton", "PA"}, {"Bethlehem", "PA"}, {"Lancaster", "PA"}, {"Harrisburg", "PA"}, {"Altoona", "PA"},
+	{"York", "PA"}, {"State College", "PA"}, {"Wilkes-Barre", "PA"}, {"Norristown", "PA"}, {"Chester", "PA"},
+	{"Bethel Park", "PA"}, {"Monroeville", "PA"}, {"Plum", "PA"}, {"Easton", "PA"}, {"Lebanon", "PA"},
+
+	// Ohio (OH)
+	{"Columbus", "OH"}, {"Cleveland", "OH"}, {"Cincinnati", "OH"}, {"Toledo", "OH"}, {"Akron", "OH"},
+	{"Dayton", "OH"}, {"Parma", "OH"}, {"Canton", "OH"}, {"Youngstown", "OH"}, {"Lorain", "OH"},
+	{"Hamilton", "OH"}, {"Springfield", "OH"}, {"Kettering", "OH"}, {"Elyria", "OH"}, {"Lakewood", "OH"},
+	{"Cuyahoga Falls", "OH"}, {"Middletown", "OH"}, {"Euclid", "OH"}, {"Newark", "OH"}, {"Mansfield", "OH"},
+
+	// Georgia (GA)
+	{"Atlanta", "GA"}, {"Augusta", "GA"}, {"Columbus", "GA"}, {"Macon", "GA"}, {"Savannah", "GA"},
+	{"Athens", "GA"}, {"Sandy Springs", "GA"}, {"Roswell", "GA"}, {"Johns Creek", "GA"}, {"Warner Robins", "GA"},
+	{"Albany", "GA"}, {"Alpharetta", "GA"}, {"Marietta", "GA"}, {"Valdosta", "GA"}, {"Smyrna", "GA"},
+	{"Dunwoody", "GA"}, {"Rome", "GA"}, {"Peachtree City", "GA"}, {"Gainesville", "GA"}, {"Brookhaven", "GA"},
+
+	// North Carolina (NC)
+	{"Charlotte", "NC"}, {"Raleigh", "NC"}, {"Greensboro", "NC"}, {"Durham", "NC"}, {"Winston-Salem", "NC"},
+	{"Fayetteville", "NC"}, {"Cary", "NC"}, {"Wilmington", "NC"}, {"High Point", "NC"}, {"Greenville", "NC"},
+	{"Asheville", "NC"}, {"Concord", "NC"}, {"Gastonia", "NC"}, {"Jacksonville", "NC"}, {"Chapel Hill", "NC"},
+	{"Rocky Mount", "NC"}, {"Huntersville", "NC"}, {"Burlington", "NC"}, {"Wilson", "NC"}, {"Kannapolis", "NC"},
+
+	// Michigan (MI)
+	{"Detroit", "MI"}, {"Grand Rapids", "MI"}, {"Warren", "MI"}, {"Sterling Heights", "MI"}, {"Ann Arbor", "MI"},
+	{"Lansing", "MI"}, {"Flint", "MI"}, {"Dearborn", "MI"}, {"Livonia", "MI"}, {"Westland", "MI"},
+	{"Troy", "MI"}, {"Farmington Hills", "MI"}, {"Kalamazoo", "MI"}, {"Wyoming", "MI"}, {"Southfield", "MI"},
+	{"Rochester Hills", "MI"}, {"Taylor", "MI"}, {"Pontiac", "MI"}, {"St. Clair Shores", "MI"}, {"Royal Oak", "MI"},
+}
+
+func randomUSCity() USCity {
+	return cities[rand.IntN(len(cities))] //nolint:gosec // this is a test,
+}

From 6aef73a5b34fe8f6796f5d46aaa246827e7da016 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Tue, 27 May 2025 13:50:31 +0800
Subject: [PATCH 07/19] More test updates

---
 .github/workflows/build-kind.yaml | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/.github/workflows/build-kind.yaml b/.github/workflows/build-kind.yaml
index 2ad7ef8..48b9b67 100644
--- a/.github/workflows/build-kind.yaml
+++ b/.github/workflows/build-kind.yaml
@@ -67,26 +67,20 @@ jobs:
       shell: bash
       run: |
         NAMESPACE=coherence-perf
+        kubectl exec -it -n $NAMESPACE perf-cluster-0 -c coherence -- /coherence-operator/utils/cohctl get caches -o wide 
         # Put the tail into the background
         POD=$(kubectl get pods -n $NAMESPACE | grep perf-go-client | awk '{print $1}') 
         kubectl logs $POD -n $NAMESPACE -f > build/_output/test-logs/perf-go-client.log 2>&1 &
         TAIL_PID=$!
-        
         # port forward the http port
         kubectl port-forward -n $NAMESPACE pod/$POD 8080:8080 > build/_output/test-logs/port-forward.log 2>&1 &
         PORT_FORWARD_PID=$!
-        
         # run curl requests
         while : ; do curl -s -v http://localhost:8080/api/schools?q=[1-2] | jq length; sleep 0.5; done > build/_output/test-logs/curl.log 2>&1 &
         CURL_PID=$!
-        
         # Start the rolling restart
         ./scripts/kind/roll-cluster.sh | tee build/_output/test-logs/rolling-restart.log
-        
-        make undeploy-test-schools
-        
         kill -9 $CURL_PID && sleep 5 && kill -9 $PORT_FORWARD_PID && kill -9 $TAIL_PID
-        
 
     - name: Shutdown
  #     if: github.event_name == 'workflow_dispatch'

From bc5e86d2c0a98d614c243fb0c2aa16d0166d9ff8 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Tue, 27 May 2025 14:02:23 +0800
Subject: [PATCH 08/19] Correct kind scripts

---
 .github/workflows/build-kind.yaml | 9 ++++++++-
 scripts/kind/test-schools.yaml    | 2 +-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-kind.yaml b/.github/workflows/build-kind.yaml
index 48b9b67..0acc041 100644
--- a/.github/workflows/build-kind.yaml
+++ b/.github/workflows/build-kind.yaml
@@ -67,7 +67,8 @@ jobs:
       shell: bash
       run: |
         NAMESPACE=coherence-perf
-        kubectl exec -it -n $NAMESPACE perf-cluster-0 -c coherence -- /coherence-operator/utils/cohctl get caches -o wide 
+        kubectl exec -it -n $NAMESPACE perf-cluster-0 -c coherence -- /coherence-operator/utils/cohctl get caches -o wide
+        make deploy-test-schools
         # Put the tail into the background
         POD=$(kubectl get pods -n $NAMESPACE | grep perf-go-client | awk '{print $1}') 
         kubectl logs $POD -n $NAMESPACE -f > build/_output/test-logs/perf-go-client.log 2>&1 &
@@ -88,6 +89,12 @@ jobs:
       run: |
         make kind-stop
 
+    - uses: actions/upload-artifact@v4
+      if: failure()
+      with:
+        name: test-output-${{ matrix.go-version }}-failure-logs
+        path: build/_output/test-logs
+
     - uses: actions/upload-artifact@v4
       with:
         name: test-output-${{ matrix.go-version }}-test-logs
diff --git a/scripts/kind/test-schools.yaml b/scripts/kind/test-schools.yaml
index 8036674..59811cc 100644
--- a/scripts/kind/test-schools.yaml
+++ b/scripts/kind/test-schools.yaml
@@ -25,7 +25,7 @@ spec:
             - name: TEST_TYPE
               value: "runTest"
             - name: COHERENCE_LOG_LEVEL
-              value: "ALL"
+              value: "DEBUG"
           resources:
             requests:
               memory: "512Mi"

From 31b248f76b89292ac51a612f9eb4078e692e198a Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Tue, 27 May 2025 14:06:36 +0800
Subject: [PATCH 09/19] More test updates

---
 .github/workflows/build-kind.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-kind.yaml b/.github/workflows/build-kind.yaml
index 0acc041..d2d31f0 100644
--- a/.github/workflows/build-kind.yaml
+++ b/.github/workflows/build-kind.yaml
@@ -60,7 +60,7 @@ jobs:
         make deploy-coherence
         make build-go-client
         make load-schools
-        mkdir -p build/_output/
+        mkdir -p build/_output/test-logs
 
     - name: Run Tests
    #   if: github.event_name == 'workflow_dispatch'

From bf1cd01f3a01efe8fca2c53b945d1c43dfd341db Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Tue, 27 May 2025 14:36:45 +0800
Subject: [PATCH 10/19] More test updates

---
 scripts/kind/roll-cluster.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/kind/roll-cluster.sh b/scripts/kind/roll-cluster.sh
index 903dff6..afd1122 100755
--- a/scripts/kind/roll-cluster.sh
+++ b/scripts/kind/roll-cluster.sh
@@ -6,7 +6,7 @@
 #
 
 set -o errexit
-maxIterations=100
+maxIterations=10
 
 if [ ! -z "$MAX_ITERATIONS" ]; then
   maxIterations=$MAX_ITERATIONS

From d16407c0f43ca7e3c225eea990362df5a74c2025 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Tue, 27 May 2025 15:04:17 +0800
Subject: [PATCH 11/19] More test updates

---
 .github/workflows/build-kind.yaml | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-kind.yaml b/.github/workflows/build-kind.yaml
index d2d31f0..34492b7 100644
--- a/.github/workflows/build-kind.yaml
+++ b/.github/workflows/build-kind.yaml
@@ -69,6 +69,23 @@ jobs:
         NAMESPACE=coherence-perf
         kubectl exec -it -n $NAMESPACE perf-cluster-0 -c coherence -- /coherence-operator/utils/cohctl get caches -o wide
         make deploy-test-schools
+        JOB_NAME=perf-go-client
+        echo "Waiting for pod from job $JOB_NAME to start running..."
+        for i in {1..30}; do
+          POD=$(kubectl get pods -n "$NAMESPACE" -l job-name="$JOB_NAME" -o jsonpath='{.items[0].metadata.name}')
+          STATUS=$(kubectl get pod "$POD" -n "$NAMESPACE" -o jsonpath='{.status.phase}')
+          echo "Pod: $POD, Status: $STATUS"
+          if [[ "$STATUS" == "Running" ]]; then
+            echo "Pod is running"
+            break
+          elif [[ "$STATUS" == "Pending" || "$STATUS" == "ContainerCreating" ]]; then
+            sleep 2
+          else
+            echo "Pod entered unexpected state: $STATUS"
+            kubectl describe pod "$POD" -n "$NAMESPACE"
+            exit 1
+          fi
+        done
         # Put the tail into the background
         POD=$(kubectl get pods -n $NAMESPACE | grep perf-go-client | awk '{print $1}') 
         kubectl logs $POD -n $NAMESPACE -f > build/_output/test-logs/perf-go-client.log 2>&1 &
@@ -81,7 +98,9 @@ jobs:
         CURL_PID=$!
         # Start the rolling restart
         ./scripts/kind/roll-cluster.sh | tee build/_output/test-logs/rolling-restart.log
-        kill -9 $CURL_PID && sleep 5 && kill -9 $PORT_FORWARD_PID && kill -9 $TAIL_PID
+        kill -9 $CURL_PID || true
+        sleep 5 && kill -9 $PORT_FORWARD_PID || true
+        kill -9 $TAIL_PID || true
 
     - name: Shutdown
  #     if: github.event_name == 'workflow_dispatch'

From f2779e7b5d14816c496fe7ae5528eef56f535812 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 28 May 2025 08:52:13 +0800
Subject: [PATCH 12/19] more updates

---
 .github/workflows/build-kind.yaml |  2 +-
 coherence/session.go              |  1 -
 test/e2e/kind/main.go             | 31 ++++++++++++++++++++-----------
 3 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/build-kind.yaml b/.github/workflows/build-kind.yaml
index 34492b7..0131ac1 100644
--- a/.github/workflows/build-kind.yaml
+++ b/.github/workflows/build-kind.yaml
@@ -94,7 +94,7 @@ jobs:
         kubectl port-forward -n $NAMESPACE pod/$POD 8080:8080 > build/_output/test-logs/port-forward.log 2>&1 &
         PORT_FORWARD_PID=$!
         # run curl requests
-        while : ; do curl -s -v http://localhost:8080/api/schools?q=[1-2] | jq length; sleep 0.5; done > build/_output/test-logs/curl.log 2>&1 &
+        while : ; do curl -s -v http://127.0.0.1:8080/api/schools?q=[1-2] | jq length; sleep 0.5; done > build/_output/test-logs/curl.log 2>&1 &
         CURL_PID=$!
         # Start the rolling restart
         ./scripts/kind/roll-cluster.sh | tee build/_output/test-logs/rolling-restart.log
diff --git a/coherence/session.go b/coherence/session.go
index 4b8048e..83a08bc 100644
--- a/coherence/session.go
+++ b/coherence/session.go
@@ -484,7 +484,6 @@ func (s *Session) GetReadyTimeout() time.Duration {
 func (s *Session) ensureConnection() error {
 	s.connectMutex.Lock()
 	defer s.connectMutex.Unlock()
-	logMessage(DEBUG, "ensureConnection()")
 
 	if s.firstConnectAttempted {
 		// We have previously tried to connect so check that the connect state is connected
diff --git a/test/e2e/kind/main.go b/test/e2e/kind/main.go
index eec0ca6..5d0f665 100644
--- a/test/e2e/kind/main.go
+++ b/test/e2e/kind/main.go
@@ -44,7 +44,7 @@ var (
 func main() {
 	var (
 		count    = 250_000
-		testType = "runTest"
+		testType = "runSchoolsTest"
 		err      error
 	)
 	if err = initializeCoherence(); err != nil {
@@ -63,7 +63,7 @@ func main() {
 	if testType == "load" {
 		err = populateSchoolsCache(count)
 	} else {
-		err = runTest()
+		err = runSchoolsTest()
 	}
 	if err != nil {
 		log.Fatalf("failed to run %s: %v", testType, err)
@@ -90,7 +90,7 @@ func initializeCoherence() error {
 	return err
 }
 
-func runTest() error {
+func runSchoolsTest() error {
 	http.HandleFunc("/api/schools", schoolsHandler)
 
 	fmt.Println("Server running on port 8080")
@@ -102,6 +102,16 @@ func runTest() error {
 func schoolsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := context.Background()
 
+	defer func() {
+		if session.IsClosed() {
+			log.Printf("Session [%s] is closed, calling initializeCoherence()", session.ID())
+			err := initializeCoherence()
+			if err != nil {
+				log.Fatalf("failed to initialize coherence: %v", err)
+			}
+		}
+	}()
+
 	switch r.Method {
 
 	case http.MethodGet:
@@ -117,14 +127,13 @@ func schoolsHandler(w http.ResponseWriter, r *http.Request) {
 
 		var people = make([]School, 0)
 
-		if session.IsClosed() {
-			log.Printf("Session [%s] is closed, calling initializeCoherence()", session.ID())
-			err := initializeCoherence()
-			if err != nil {
-				log.Printf("failed to initialize coherence: %v", err)
-				http.Error(w, "not connected", http.StatusInternalServerError)
-				return
-			}
+		log.Println("Health check via size()")
+		_, err := schoolsCache.Size(ctx)
+		if err != nil {
+			session.Close()
+			log.Printf("failed to do health check: %v", err)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
 		}
 
 		for ch := range schoolsCache.ValuesFilter(ctx, marketSegmentFilter) {

From 258ba1f203b4608d3eb60a4d0d5fdb5b4282f513 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 28 May 2025 09:15:11 +0800
Subject: [PATCH 13/19] Tweak github actions

---
 .github/workflows/build-kind.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-kind.yaml b/.github/workflows/build-kind.yaml
index 0131ac1..fbfcd85 100644
--- a/.github/workflows/build-kind.yaml
+++ b/.github/workflows/build-kind.yaml
@@ -91,8 +91,10 @@ jobs:
         kubectl logs $POD -n $NAMESPACE -f > build/_output/test-logs/perf-go-client.log 2>&1 &
         TAIL_PID=$!
         # port forward the http port
-        kubectl port-forward -n $NAMESPACE pod/$POD 8080:8080 > build/_output/test-logs/port-forward.log 2>&1 &
+        echo "Pod: $POD"
+        kubectl port-forward -n $NAMESPACE pod/${POD} 8080:8080 > build/_output/test-logs/port-forward.log 2>&1 &
         PORT_FORWARD_PID=$!
+        sleep 5
         # run curl requests
         while : ; do curl -s -v http://127.0.0.1:8080/api/schools?q=[1-2] | jq length; sleep 0.5; done > build/_output/test-logs/curl.log 2>&1 &
         CURL_PID=$!

From 9dc412b18a278ab9f59029adac03b8aeafce360e Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 28 May 2025 09:48:58 +0800
Subject: [PATCH 14/19] further script updates

---
 .github/workflows/build-kind.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-kind.yaml b/.github/workflows/build-kind.yaml
index fbfcd85..e2151cb 100644
--- a/.github/workflows/build-kind.yaml
+++ b/.github/workflows/build-kind.yaml
@@ -94,9 +94,10 @@ jobs:
         echo "Pod: $POD"
         kubectl port-forward -n $NAMESPACE pod/${POD} 8080:8080 > build/_output/test-logs/port-forward.log 2>&1 &
         PORT_FORWARD_PID=$!
-        sleep 5
+        echo "Sleep 10..." 
+        sleep 10
         # run curl requests
-        while : ; do curl -s -v http://127.0.0.1:8080/api/schools?q=[1-2] | jq length; sleep 0.5; done > build/_output/test-logs/curl.log 2>&1 &
+        while : ; do curl -s -v http://127.0.0.1:8080/api/schools?q=[1-2] | jq length || true ; sleep 0.5; done > build/_output/test-logs/curl.log 2>&1 &
         CURL_PID=$!
         # Start the rolling restart
         ./scripts/kind/roll-cluster.sh | tee build/_output/test-logs/rolling-restart.log

From 8f400730d5f3dd1bb0366c0cc4e03502f5552d7d Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 28 May 2025 10:01:35 +0800
Subject: [PATCH 15/19] Fixup perms of docker file

---
 test/e2e/kind/Dockerfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/e2e/kind/Dockerfile b/test/e2e/kind/Dockerfile
index 0c5cf5c..4130d95 100644
--- a/test/e2e/kind/Dockerfile
+++ b/test/e2e/kind/Dockerfile
@@ -6,9 +6,10 @@
 
 FROM scratch
 
-COPY runner /files/runner
+COPY --chown=1000:1000 runner /files/runner
 
-EXPOSE 8080
+USER 1000:1000
 
+EXPOSE 8080
 ENTRYPOINT ["/files/runner"]
 CMD ["-h"]

From e563fb74f69779fd2ea218ea232c3f305a48c587 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 28 May 2025 10:08:42 +0800
Subject: [PATCH 16/19] refine docker image

---
 test/e2e/kind/Dockerfile | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/test/e2e/kind/Dockerfile b/test/e2e/kind/Dockerfile
index 4130d95..f5330cb 100644
--- a/test/e2e/kind/Dockerfile
+++ b/test/e2e/kind/Dockerfile
@@ -4,12 +4,18 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at
 # http://oss.oracle.com/licenses/upl.
 
+FROM alpine AS builder
+
+COPY runner /runner
+RUN chmod 0555 /runner
+
 FROM scratch
 
-COPY --chown=1000:1000 runner /files/runner
+COPY --chown=1000:1000 --from=builder /runner /files/runner
 
 USER 1000:1000
 
 EXPOSE 8080
+
 ENTRYPOINT ["/files/runner"]
-CMD ["-h"]
+CMD ["-h"]
\ No newline at end of file

From 01584f0b0c745cdf89736d1b7c9052faef37af97 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 28 May 2025 11:12:38 +0800
Subject: [PATCH 17/19] Add iterations parameter to script

---
 .github/workflows/build-kind.yaml | 7 +++++++
 scripts/kind/test-schools.yaml    | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-kind.yaml b/.github/workflows/build-kind.yaml
index e2151cb..6416634 100644
--- a/.github/workflows/build-kind.yaml
+++ b/.github/workflows/build-kind.yaml
@@ -17,6 +17,10 @@ on:
         description: "Go version (comma-separated for matrix)"
         required: false
         default: "1.23.x,1.24.x"
+      max-iterations:
+        description: "Maximum number of iterations"
+        required: false
+        default: "10"
 
 jobs:
   build:
@@ -65,7 +69,10 @@ jobs:
     - name: Run Tests
    #   if: github.event_name == 'workflow_dispatch'
       shell: bash
+      env:
+        MAX_ITERATIONS: ${{ github.event.inputs.max-iterations }}
       run: |
+        export MAX_ITERATIONS
         NAMESPACE=coherence-perf
         kubectl exec -it -n $NAMESPACE perf-cluster-0 -c coherence -- /coherence-operator/utils/cohctl get caches -o wide
         make deploy-test-schools
diff --git a/scripts/kind/test-schools.yaml b/scripts/kind/test-schools.yaml
index 59811cc..2c41751 100644
--- a/scripts/kind/test-schools.yaml
+++ b/scripts/kind/test-schools.yaml
@@ -21,7 +21,7 @@ spec:
             - name: COHERENCE_SERVER_ADDRESS
               value: "perf-cluster-grpc:1408"
             - name: COHERENCE_READY_TIMEOUT
-              value: "15000"
+              value: "30000"
             - name: TEST_TYPE
               value: "runTest"
             - name: COHERENCE_LOG_LEVEL

From 43682686c2949ae8e1a9c8469a344ace3845e5c5 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 28 May 2025 14:11:58 +0800
Subject: [PATCH 18/19] Final updates

---
 coherence/session.go  | 1 -
 test/e2e/kind/main.go | 5 ++---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/coherence/session.go b/coherence/session.go
index 83a08bc..b153fd0 100644
--- a/coherence/session.go
+++ b/coherence/session.go
@@ -678,7 +678,6 @@ func waitForReady(s *Session) error {
 		if state == connectivity.Ready {
 			return nil
 		}
-		logMessage(DEBUG, "state=%v", state)
 
 		if !messageLogged {
 			logMessage(INFO, "Session [%s] State is %v, waiting until ready timeout of %v for valid connection",
diff --git a/test/e2e/kind/main.go b/test/e2e/kind/main.go
index 5d0f665..94c1fb4 100644
--- a/test/e2e/kind/main.go
+++ b/test/e2e/kind/main.go
@@ -127,9 +127,8 @@ func schoolsHandler(w http.ResponseWriter, r *http.Request) {
 
 		var people = make([]School, 0)
 
-		log.Println("Health check via size()")
-		_, err := schoolsCache.Size(ctx)
-		if err != nil {
+		ready, err := schoolsCache.IsReady(ctx)
+		if err != nil || !ready {
 			session.Close()
 			log.Printf("failed to do health check: %v", err)
 			http.Error(w, err.Error(), http.StatusInternalServerError)

From 6a30ab7b986f3574162cb0bbdbd87f2c8f049f7e Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 28 May 2025 14:12:20 +0800
Subject: [PATCH 19/19] Update version in Makefile

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 443be02..a22fca0 100644
--- a/Makefile
+++ b/Makefile
@@ -8,7 +8,7 @@
 # ----------------------------------------------------------------------------------------------------------------------
 
 # This is the version of the coherence-go-client
-VERSION ?=2.3.0-rc1
+VERSION ?=2.3.0-rc2
 CURRDIR := $(shell pwd)
 USER_ID := $(shell echo "`id -u`:`id -g`")