Add additional kind tests

.github/workflows/build-kind.yaml

@@ -0,0 +1,101 @@
+# Copyright 2025 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# https://oss.oracle.com/licenses/upl.
+
+# ---------------------------------------------------------------------------
+# Coherence Go Client GitHub Actions - Kind Tests
+# ---------------------------------------------------------------------------
+name: CI Kind Tests
+
+on:
+  push:
+    branches:
+      - '*'
+  workflow_dispatch:
+    inputs:
+      go-version:
+        description: "Go version (comma-separated for matrix)"
+        required: false
+        default: "1.23.x,1.24.x"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    strategy:
+      fail-fast: false
+      matrix:
+        go-version:
+          - 1.24.x
+
+#   Checkout the source, we need a depth of zero to fetch all of the history otherwise
+#   the copyright check cannot work out the date of the files from Git.
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Cache Go Modules
+    #  if: github.event_name == 'workflow_dispatch'
+      uses: actions/cache@v4
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-mods-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-mods-
+
+    - name: Set up Go
+  #    if: github.event_name == 'workflow_dispatch'
+      uses: actions/setup-go@v5
+      with:
+        go-version: '${{ matrix.go-version }}'
+
+    - name: Create Kind Cluster
+ #     if: github.event_name == 'workflow_dispatch'
+      shell: bash
+      run: |
+        make kind
+        make create-namespace deploy-operator
+        make deploy-coherence
+        make build-go-client
+        make load-schools
+        mkdir -p build/_output/
+
+    - name: Run Tests
+   #   if: github.event_name == 'workflow_dispatch'
+      shell: bash
+      run: |
+        NAMESPACE=coherence-perf
+        # Put the tail into the background
+        POD=$(kubectl get pods -n $NAMESPACE | grep perf-go-client | awk '{print $1}') 
+        kubectl logs $POD -n $NAMESPACE -f > build/_output/test-logs/perf-go-client.log 2>&1 &
+        TAIL_PID=$!
+
+        # port forward the http port
+        kubectl port-forward -n $NAMESPACE pod/$POD 8080:8080 > build/_output/test-logs/port-forward.log 2>&1 &
+        PORT_FORWARD_PID=$!
+
+        # run curl requests
+        while : ; do curl -s -v http://localhost:8080/api/schools?q=[1-2] | jq length; sleep 0.5; done > build/_output/test-logs/curl.log 2>&1 &
+        CURL_PID=$!
+
+        # Start the rolling restart
+        ./scripts/kind/roll-cluster.sh | tee build/_output/test-logs/rolling-restart.log
+
+        make undeploy-test-schools
+
+        kill -9 $CURL_PID && sleep 5 && kill -9 $PORT_FORWARD_PID && kill -9 $TAIL_PID
+
+
+    - name: Shutdown
+ #     if: github.event_name == 'workflow_dispatch'
+      shell: bash
+      run: |
+        make kind-stop
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: test-output-${{ matrix.go-version }}-test-logs
+        path: build/_output/test-logs
+

.gitignore

@@ -41,3 +41,4 @@ certs/
 release/
 etc/
 coherence/coherence.test
+runner

Makefile

@@ -178,6 +178,7 @@ copyright: getcopyright ## Check copyright headers
 	  -X proto/ \
 	  -X /Dockerfile \
 	  -X .Dockerfile \
+	  -X runner \
 	  -X go.sum \
 	  -X HEADER.txt \
 	  -X .iml \
@@ -277,6 +278,111 @@ show-docs:   ## Show the Documentation
 trivy-scan: gettrivy ## Scan the CLI using trivy
 	$(TOOLS_BIN)/trivy fs --cache-dir ${TRIVY_CACHE} --exit-code 1 --skip-dirs "./java" .
 
+# ======================================================================================================================
+# Targets related to running KinD clusters for testing
+# ======================================================================================================================
+##@ KinD
+
+KIND_CLUSTER     ?= go-client
+KIND_IMAGE       ?= "kindest/node:v1.33.0@sha256:91e9ed777db80279c22d1d1068c091b899b2078506e4a0f797fbf6e397c0b0b2"
+KIND_SCRIPTS     := ./scripts/kind
+NAMESPACE        ?= coherence-perf
+OPERATOR_VERSION ?= v3.5.0
+COHERENCE_IMAGE  ?= ghcr.io/oracle/coherence-ce:14.1.2-0-2-java17
+INITIAL_HEAP     ?= 1g
+GO_CLIENT_ARCH   ?= amd64
+GO_IMAGE         ?= perf-go-client:1.0.0
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Start a Kind cluster
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: kind
+kind:   ## Run a default KinD cluster
+	kind create cluster --name $(KIND_CLUSTER) --wait 10m --config $(KIND_SCRIPTS)/kind-config.yaml --image $(KIND_IMAGE)
+	$(KIND_SCRIPTS)/kind-label-node.sh
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Stop and delete the Kind cluster
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: kind-stop
+kind-stop:   ## Stop and delete the KinD cluster
+	kind delete cluster --name $(KIND_CLUSTER)
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Deploy Coherence Operator
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: deploy-operator
+deploy-operator:   ## Deploy the Coherence Operator
+	kubectl apply -f https://github.com/oracle/coherence-operator/releases/download/$(OPERATOR_VERSION)/coherence-operator.yaml
+	kubectl -n coherence wait --timeout=300s --for condition=available deployment/coherence-operator-controller-manager
+
+# ----------------------------------------------------------------------------------------------------------------------
+# UnDeploy Coherence Operator
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: undeploy-operator
+undeploy-operator:   ## UnDeploy the Coherence Operator
+	kubectl delete -f https://github.com/oracle/coherence-operator/releases/download/$(OPERATOR_VERSION)/coherence-operator.yaml || true
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Deploy Coherence Cluster
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: deploy-coherence
+deploy-coherence:   ## Deploy the Coherence Cluster
+	envsubst <  $(KIND_SCRIPTS)/coherence-cluster.yaml | kubectl apply -n $(NAMESPACE) -f -
+	sleep 5
+	kubectl -n $(NAMESPACE) wait --timeout=300s --for condition=Ready coherence/perf-cluster
+
+#-----------------------------------------------------------------------------------------------------------------------
+# Deploy Coherence Cluster
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: build-go-client
+build-go-client:   ## Make Go Client
+	cd test/e2e/kind && CGO_ENABLED=0 GOOS=linux GOARCH=$(GO_CLIENT_ARCH) GO111MODULE=on go build -trimpath -o runner . && docker build --no-cache -t $(GO_IMAGE) .
+	kind --name $(KIND_CLUSTER) load docker-image $(GO_IMAGE)
+
+#-----------------------------------------------------------------------------------------------------------------------
+# Load Schools Data
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: load-schools
+load-schools:   ## Load Schools
+	kubectl -n $(NAMESPACE) apply -f $(KIND_SCRIPTS)/load-schools.yaml
+	kubectl wait -n $(NAMESPACE) --timeout=1200s --for condition=Complete job/go-perf-load-schools
+	kubectl -n $(NAMESPACE) delete -f $(KIND_SCRIPTS)/load-schools.yaml || true
+
+#-----------------------------------------------------------------------------------------------------------------------
+# Test Schools
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: deploy-test-schools
+deploy-test-schools:   ## Deploy Test Schools
+	kubectl -n $(NAMESPACE) apply -f $(KIND_SCRIPTS)/test-schools.yaml
+
+#-----------------------------------------------------------------------------------------------------------------------
+# Stop Schools Test
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: undeploy-test-schools
+undeploy-test-schools:   ## Undeploy Test Schools
+	kubectl -n $(NAMESPACE) delete -f $(KIND_SCRIPTS)/test-schools.yaml
+
+# ----------------------------------------------------------------------------------------------------------------------
+# UnDeploy Coherence Cluster
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: undeploy-coherence
+undeploy-coherence:   ## UnDeploy the Coherence Cluster
+	kubectl delete -n $(NAMESPACE) -f $(KIND_SCRIPTS)/coherence-cluster.yaml || true
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Create Perf namespace
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: create-namespace
+create-namespace:   ## Create the perf test namespace
+	kubectl create namespace $(NAMESPACE)
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Delete Perf namespace
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: delete-namespace
+delete-namespace:   ## Create the perf test namespace
+	kubectl delete namespace $(NAMESPACE) || true
 
 # ======================================================================================================================
 # Test targets
@@ -344,7 +450,6 @@ test-v1-base: test-clean test gotestsum $(BUILD_PROPS) ## Run e2e tests with Coh
 test-examples: test-clean gotestsum $(BUILD_PROPS) ## Run examples tests with Coherence
 	./scripts/run-test-examples.sh
 
-
 # ----------------------------------------------------------------------------------------------------------------------
 # Startup cluster members via docker compose
 # ----------------------------------------------------------------------------------------------------------------------

coherence/common.go

@@ -57,6 +57,8 @@ const (
 	integerMaxValue = 2147483647
 
 	ListenAll InvalidationStrategyType = 0
+
+	panicWarning = "recovered from panic, possible connection closed while received channel data: %v"
 )
 
 var (
@@ -636,6 +638,14 @@ func executeGetAll[K comparable, V any](ctx context.Context, bc *baseClient[K, V
 	}
 
 	go func() {
+		defer func() {
+			// catch panic of closed channel read in rare circumstances
+			if r := recover(); r != nil {
+				logMessage(WARNING, panicWarning, r)
+				return
+			}
+		}()
+
 		if cancel != nil {
 			defer cancel()
 		}
@@ -1185,6 +1195,14 @@ func executeInvokeAllFilterOrKeys[K comparable, V any, R any](ctx context.Contex
 	}
 
 	go func() {
+		defer func() {
+			// catch panic of closed channel read in rare circumstances
+			if r := recover(); r != nil {
+				logMessage(WARNING, panicWarning, r)
+				return
+			}
+		}()
+
 		if cancel != nil {
 			defer cancel()
 		}
@@ -1264,10 +1282,18 @@ func executeKeySet[K comparable, V any](ctx context.Context, bc *baseClient[K, V
 	}
 
 	go func() {
+		defer func() {
+			// catch panic of closed channel read in rare circumstances
+			if r := recover(); r != nil {
+				logMessage(WARNING, panicWarning, r)
+				return
+			}
+		}()
+
 		for {
 			result, err1 := iter.Next()
 
-			if err1 == ErrDone {
+			if errors.Is(err1, ErrDone) {
 				close(ch)
 				return
 			} else if err1 != nil {
@@ -1307,6 +1333,14 @@ func executeKeySetFilter[K comparable, V any](ctx context.Context, bc *baseClien
 	}
 
 	go func() {
+		defer func() {
+			// catch panic of closed channel read in rare circumstances
+			if r := recover(); r != nil {
+				logMessage(WARNING, panicWarning, r)
+				return
+			}
+		}()
+
 		if cancel != nil {
 			defer cancel()
 		}
@@ -1840,10 +1874,18 @@ func executeEntrySet[K comparable, V any](ctx context.Context, bc *baseClient[K,
 	}
 
 	go func() {
+		defer func() {
+			// catch panic of closed channel read in rare circumstances
+			if r := recover(); r != nil {
+				logMessage(WARNING, panicWarning, r)
+				return
+			}
+		}()
+
 		for {
 			result, err1 := iter.Next()
 
-			if err1 == ErrDone {
+			if errors.Is(err1, ErrDone) {
 				close(ch)
 				return
 			} else if err1 != nil {
@@ -1892,6 +1934,14 @@ func executeEntrySetFilter[K comparable, V any, E any](ctx context.Context, bc *
 	}
 
 	go func() {
+		defer func() {
+			// catch panic of closed channel read in rare circumstances
+			if r := recover(); r != nil {
+				logMessage(WARNING, panicWarning, r)
+				return
+			}
+		}()
+
 		if cancel != nil {
 			defer cancel()
 		}
@@ -1986,6 +2036,14 @@ func executeValues[K comparable, V any, E any](ctx context.Context, bc *baseClie
 	}
 
 	go func() {
+		defer func() {
+			// catch panic of closed channel read in rare circumstances
+			if r := recover(); r != nil {
+				logMessage(WARNING, panicWarning, r)
+				return
+			}
+		}()
+
 		if cancel != nil {
 			defer cancel()
 		}
@@ -2056,10 +2114,18 @@ func executeValuesNoFilter[K comparable, V any](ctx context.Context, bc *baseCli
 	}
 
 	go func() {
+		defer func() {
+			// catch panic of closed channel read in rare circumstances
+			if r := recover(); r != nil {
+				logMessage(WARNING, panicWarning, r)
+				return
+			}
+		}()
+
 		for {
 			result, err1 := iter.Next()
 
-			if err1 == ErrDone {
+			if errors.Is(err1, ErrDone) {
 				close(ch)
 				return
 			} else if err1 != nil {