From 97bd12faf0d7b801b016a66c8361cfd8cb34da46 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 30 Apr 2025 12:14:33 +0800
Subject: [PATCH 1/9] Initial perf tests

---
 .github/workflows/build-perf.yaml |  76 ++++++++++++++++
 Makefile                          |  31 +++++--
 scripts/perf-cluster.sh           | 105 ++++++++++++++++++++++
 test/e2e/perf/run_test.go         |  23 +++++
 test/e2e/perf/suite_test.go       | 141 ++++++++++++++++++++++++++++++
 5 files changed, 367 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/build-perf.yaml
 create mode 100755 scripts/perf-cluster.sh
 create mode 100644 test/e2e/perf/run_test.go
 create mode 100644 test/e2e/perf/suite_test.go

diff --git a/.github/workflows/build-perf.yaml b/.github/workflows/build-perf.yaml
new file mode 100644
index 0000000..897b41e
--- /dev/null
+++ b/.github/workflows/build-perf.yaml
@@ -0,0 +1,76 @@
+# Copyright 2022, 2025 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# https://oss.oracle.com/licenses/upl.
+
+# ---------------------------------------------------------------------------
+# Coherence Go Client GitHub Actions CI Perf Tests
+# ---------------------------------------------------------------------------
+name: CI Perf Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches-ignore:
+      - gh-pages
+  schedule:
+    # Every day at midnight
+    - cron: '0 0 * * *'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        coherenceVersion:
+          - 14.1.2-0-2
+          - 25.03.1
+        go-version:
+          - 1.23.x
+          - 1.24.x
+
+
+#   Checkout the source, we need a depth of zero to fetch all of the history otherwise
+#   the copyright check cannot work out the date of the files from Git.
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up JDK
+      uses: actions/setup-java@v4
+      with:
+        java-version: '17'
+        distribution: 'zulu'
+
+    - name: Cache Go Modules
+      uses: actions/cache@v4
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-mods-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-mods-
+
+    - name: Cache Maven packages
+      uses: actions/cache@v4
+      with:
+        path: ~/.m2
+        key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+        restore-keys: ${{ runner.os }}-m2
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: '${{ matrix.go-version }}'
+
+    - name: Run Perf Test
+      shell: bash
+      run: |
+        curl -sL https://raw.githubusercontent.com/oracle/coherence-cli/main/scripts/install.sh | bash
+        COHERENCE_VERSION=${{ coherenceVersion }} make test-perf
+       
+    - uses: actions/upload-artifact@v4
+      if: failure()
+      with:
+        name: test-output-${{ matrix.go-version }}-${{ coherenceVersion }}
+        path: build/_output/test-logs
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 3b4936d..91766fa 100644
--- a/Makefile
+++ b/Makefile
@@ -34,7 +34,7 @@ MVN_VERSION ?= 1.0.0
 SHELL := /bin/bash
 
 # Coherence CE version to run base tests against
-COHERENCE_VERSION ?= 22.06.11
+COHERENCE_VERSION ?= 22.06.12
 COHERENCE_GROUP_ID ?= com.oracle.coherence.ce
 COHERENCE_WKA1 ?= server1
 COHERENCE_WKA2 ?= server1
@@ -207,8 +207,8 @@ golangci: $(TOOLS_BIN)/golangci-lint ## Go code review
 .PHONY: generate-proto
 generate-proto: $(TOOLS_BIN)/protoc ## Generate Proto Files
 	mkdir -p $(PROTO_DIR) || true
-	curl -o $(PROTO_DIR)/services.proto https://raw.githubusercontent.com/oracle/coherence/22.06.11/prj/coherence-grpc/src/main/proto/services.proto
-	curl -o $(PROTO_DIR)/messages.proto https://raw.githubusercontent.com/oracle/coherence/22.06.11/prj/coherence-grpc/src/main/proto/messages.proto
+	curl -o $(PROTO_DIR)/services.proto https://raw.githubusercontent.com/oracle/coherence/22.06.12/prj/coherence-grpc/src/main/proto/services.proto
+	curl -o $(PROTO_DIR)/messages.proto https://raw.githubusercontent.com/oracle/coherence/22.06.12/prj/coherence-grpc/src/main/proto/messages.proto
 	echo "" >> $(PROTO_DIR)/services.proto
 	echo "" >> $(PROTO_DIR)/messages.proto
 	echo 'option go_package = "github.com/oracle/coherence-go-client/proto";' >> $(PROTO_DIR)/services.proto
@@ -221,11 +221,11 @@ generate-proto: $(TOOLS_BIN)/protoc ## Generate Proto Files
 .PHONY: generate-proto-v1
 generate-proto-v1: $(TOOLS_BIN)/protoc ## Generate Proto Files v1
 	mkdir -p $(PROTOV1_DIR) || true
-	curl -o $(PROTOV1_DIR)/proxy_service_messages_v1.proto https://raw.githubusercontent.com/oracle/coherence/25.03/prj/coherence-grpc/src/main/proto/proxy_service_messages_v1.proto
-	curl -o $(PROTOV1_DIR)/proxy_service_v1.proto https://raw.githubusercontent.com/oracle/coherence/25.03/prj/coherence-grpc/src/main/proto/proxy_service_v1.proto
-	curl -o $(PROTOV1_DIR)/common_messages_v1.proto https://raw.githubusercontent.com/oracle/coherence/25.03/prj/coherence-grpc/src/main/proto/common_messages_v1.proto
-	curl -o $(PROTOV1_DIR)/cache_service_messages_v1.proto https://raw.githubusercontent.com/oracle/coherence/25.03/prj/coherence-grpc/src/main/proto/cache_service_messages_v1.proto
-	curl -o $(PROTOV1_DIR)/queue_service_messages_v1.proto https://raw.githubusercontent.com/oracle/coherence/25.03/prj/coherence-grpc/src/main/proto/queue_service_messages_v1.proto
+	curl -o $(PROTOV1_DIR)/proxy_service_messages_v1.proto https://raw.githubusercontent.com/oracle/coherence/25.03.1/prj/coherence-grpc/src/main/proto/proxy_service_messages_v1.proto
+	curl -o $(PROTOV1_DIR)/proxy_service_v1.proto https://raw.githubusercontent.com/oracle/coherence/25.03.1/prj/coherence-grpc/src/main/proto/proxy_service_v1.proto
+	curl -o $(PROTOV1_DIR)/common_messages_v1.proto https://raw.githubusercontent.com/oracle/coherence/25.03.1/prj/coherence-grpc/src/main/proto/common_messages_v1.proto
+	curl -o $(PROTOV1_DIR)/cache_service_messages_v1.proto https://raw.githubusercontent.com/oracle/coherence/25.03.1/prj/coherence-grpc/src/main/proto/cache_service_messages_v1.proto
+	curl -o $(PROTOV1_DIR)/queue_service_messages_v1.proto https://raw.githubusercontent.com/oracle/coherence/25.03.1/prj/coherence-grpc/src/main/proto/queue_service_messages_v1.proto
 	echo "" >> $(PROTOV1_DIR)/proxy_service_messages_v1.proto
 	echo "" >> $(PROTOV1_DIR)/proxy_service_v1.proto
 	echo "" >> $(PROTOV1_DIR)/common_messages_v1.proto
@@ -389,6 +389,19 @@ test-discovery: test-clean gotestsum $(BUILD_PROPS) ## Run Discovery tests with
 	  -- $(GO_TEST_FLAGS) -v  ./test/e2e/discovery/...
 	make test-coherence-shutdown
 
+# ----------------------------------------------------------------------------------------------------------------------
+# Executes the Go perf tests for standalone Coherence
+# ----------------------------------------------------------------------------------------------------------------------
+.PHONY: test-perf
+test-perf: test-clean gotestsum $(BUILD_PROPS) ## Run Discovery tests with Coherence
+	./scripts/perf-cluster.sh $(TEST_LOGS_DIR)/cli $(COHERENCE_VERSION) stop || true
+	mkdir -p $(TEST_LOGS_DIR)/cli
+	./scripts/perf-cluster.sh $(TEST_LOGS_DIR)/cli $(COHERENCE_VERSION) start
+	CGO_ENABLED=0 $(GOTESTSUM) --format testname --junitfile $(TEST_LOGS_DIR)/cohctl-test-perf.xml \
+	  -- $(GO_TEST_FLAGS) -v  ./test/e2e/perf/...
+	./scripts/perf-cluster.sh $(TEST_LOGS_DIR)/cli $(COHERENCE_VERSION) stop || true
+	rm -rf $(TEST_LOGS_DIR)/cli/*
+
 # ----------------------------------------------------------------------------------------------------------------------
 # Executes the Go resolver tests for standalone Coherence
 # ----------------------------------------------------------------------------------------------------------------------
@@ -413,7 +426,7 @@ test-resolver-cluster: test-clean gotestsum $(BUILD_PROPS) ## Run Resolver tests
 # ----------------------------------------------------------------------------------------------------------------------
 $(TOOLS_BIN)/golangci-lint:
 	@mkdir -p $(TOOLS_BIN)
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TOOLS_BIN) v1.61.0
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TOOLS_BIN) v1.64.8
 
 
 # ----------------------------------------------------------------------------------------------------------------------
diff --git a/scripts/perf-cluster.sh b/scripts/perf-cluster.sh
new file mode 100755
index 0000000..fba00ca
--- /dev/null
+++ b/scripts/perf-cluster.sh
@@ -0,0 +1,105 @@
+#!/bin/bash
+
+#
+# Copyright (c) 2025 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# https://oss.oracle.com/licenses/upl.
+#
+
+# Run Performance Test
+# environment variables COM accepted
+# Arguments:
+# 1 - directory for cohctl config
+# 2 - coherence version
+# 3 - start or stop
+pwd
+
+if [ $# -ne 3 ] ; then
+   echo "Usage: $0 directory Coherence-Version [start|stop]"
+   exit
+fi
+
+CONFIG_DIR=$1
+VERSION=$2
+COMMAND=$3
+
+if [ ! -d $CONFIG_DIR ]; then
+  echo "${CONFIG_DIR} is not a directory"
+  exit 1
+fi
+
+DIR=`pwd`
+OUTPUT=`mktemp`
+
+mkdir -p ${CONFIG_DIR}
+trap "rm -rf ${OUTPUT}" EXIT SIGINT
+
+echo
+echo "Config Dir: ${CONFIG_DIR}"
+echo "Version:    ${VERSION}"
+echo "Commercial: ${COM}"
+echo
+
+type cohctl
+ret=$?
+if [ $ret -ne 0 ]; then
+  echo "cohctl must be in path"
+  exit 1
+fi
+
+# Build the Java project so we get any deps downloaded
+
+COHERENCE_GROUP_ID=com.oracle.coherence.ce
+if [ ! -z "$COM" ] ; then
+  COHERENCE_GROUP_ID=com.oracle.coherence
+fi
+
+# Default command
+COHCTL="cohctl --config-dir ${CONFIG_DIR}"
+
+function pause() {
+   echo "sleeping..."
+   sleep 5
+}
+
+function message() {
+    echo "========================================================="
+    echo "$*"
+}
+
+function save_logs() {
+    mkdir -p build/_output/test-logs
+    cp ${CONFIG_DIR}/logs/local/*.log build/_output/test-logs || true
+}
+
+function runCommand() {
+    echo "========================================================="
+    echo "Running command: cohctl $*"
+    $COHCTL $* > $OUTPUT 2>&1
+    ret=$?
+    cat $OUTPUT
+    if [ $ret -ne 0 ] ; then
+      echo "Command failed"
+      # copy the log files
+      save_logs
+      exit 1
+    fi
+}
+
+runCommand version
+runCommand set debug on
+
+if [ "${COMMAND}" == "start" ]; then
+  # Create a cluster
+  message "Create Cluster"
+  runCommand create cluster local -y -v $VERSION $COM -S com.tangosol.net.Coherence -a coherence-grpc,coherence-grpc-proxy --machine machine1 -M 2g
+  runCommand monitor health -n localhost:7574 -I -T 120 -w
+elif [ "${COMMAND}" == "stop" ]; then
+  runCommand stop cluster local -y
+  runCommand remove cluster local -y
+elif [ "${COMMAND}" == "status" ]; then
+  runCommand get members
+else
+  echo "Invalid command ${COMMAND}"
+  exit 1
+fi
diff --git a/test/e2e/perf/run_test.go b/test/e2e/perf/run_test.go
new file mode 100644
index 0000000..7cf7ce0
--- /dev/null
+++ b/test/e2e/perf/run_test.go
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates.
+ * Licensed under the Universal Permissive License v 1.0 as shown at
+ * https://oss.oracle.com/licenses/upl.
+ */
+
+package perf
+
+import (
+	"github.com/onsi/gomega"
+	"testing"
+)
+
+// TestPerformance1 tests for discovery
+func TestPerformance1(t *testing.T) {
+	var (
+		g = gomega.NewWithT(t)
+	)
+
+	size, err := config.Students.Size(ctx)
+	g.Expect(err).To(gomega.BeNil())
+	g.Expect(size).To(gomega.Equal(maxStudents))
+}
diff --git a/test/e2e/perf/suite_test.go b/test/e2e/perf/suite_test.go
new file mode 100644
index 0000000..f4e7e1c
--- /dev/null
+++ b/test/e2e/perf/suite_test.go
@@ -0,0 +1,141 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates.
+ * Licensed under the Universal Permissive License v 1.0 as shown at
+ * https://oss.oracle.com/licenses/upl.
+ */
+
+package perf
+
+import (
+	"context"
+	"fmt"
+	"github.com/oracle/coherence-go-client/v2/coherence"
+	"log"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+)
+
+type Student struct {
+	ID      int    `json:"id"`
+	Name    string `json:"name"`
+	Address string `json:"address"`
+	Course  string `json:"course"`
+	Country string `json:"country"`
+}
+
+type Config struct {
+	Session  *coherence.Session
+	Students coherence.NamedCache[int, Student]
+}
+
+var (
+	ctx = context.Background()
+	//nolint:gosec // just a test
+	rnd    = rand.New(rand.NewSource(time.Now().UnixNano()))
+	config = Config{}
+)
+
+const (
+	maxStudents = 2_000_000
+)
+
+// The entry point for the test suite
+func TestMain(m *testing.M) {
+	var (
+		err  error
+		size int
+	)
+
+	log.Println("Connecting to cluster")
+	config, err = InitializeCoherence(ctx, "coherence:///localhost:7574")
+	if err != nil {
+		errorAndExit("connecting to cluster", err)
+	}
+
+	err = config.Students.Clear(ctx)
+	if err != nil {
+		errorAndExit("unable to clear", err)
+	}
+
+	log.Println("Populating cache with", maxStudents, "students")
+	if populateCache(config.Students, maxStudents) != nil {
+		errorAndExit("failed to populate cache", err)
+	}
+
+	size, err = config.Students.Size(ctx)
+	if err != nil || size != maxStudents {
+		errorAndExit("size not correct", err)
+	}
+	log.Println("Cache size is", size)
+
+	exitCode := m.Run()
+
+	fmt.Printf("Tests completed with return code %d\n", exitCode)
+
+	os.Exit(exitCode)
+}
+
+func errorAndExit(message string, err error) {
+	fmt.Println(message, err)
+	os.Exit(1)
+}
+
+func InitializeCoherence(ctx context.Context, address string) (Config, error) {
+	var (
+		err error
+	)
+
+	// create a new Session to the default gRPC port of 1408 using plain text
+	config.Session, err = coherence.NewSession(ctx, coherence.WithPlainText(), coherence.WithAddress(address))
+	if err != nil {
+		return config, err
+	}
+
+	config.Students, err = coherence.GetNamedCache[int, Student](config.Session, "students")
+	if err != nil {
+		return config, err
+	}
+
+	return config, nil
+}
+
+func populateCache(cache coherence.NamedMap[int, Student], count int) error {
+	var (
+		buffer    = make(map[int]Student)
+		err       error
+		courses   = []string{"C1", "C2", "C3", "C4"}
+		countries = []string{"Australia", "USA", "Canada", "Mexico"}
+		batchSize = 10_000
+	)
+
+	for i := 1; i <= count; i++ {
+		buffer[i] = Student{
+			ID:      i,
+			Name:    fmt.Sprintf("student%d", i),
+			Address: fmt.Sprintf("address%d", i),
+			Country: randomize(countries),
+			Course:  randomize(courses),
+		}
+		if i%batchSize == 0 {
+			err = cache.PutAll(ctx, buffer)
+			if err != nil {
+				return err
+			}
+			buffer = make(map[int]Student)
+		}
+	}
+	if len(buffer) > 0 {
+		return cache.PutAll(ctx, buffer)
+	}
+
+	return nil
+}
+
+func randomize(arr []string) string {
+	if len(arr) == 0 {
+		return ""
+	}
+	return arr[rnd.Intn(len(arr))]
+}

From a740e2691b0fd2e0cb1af73fa520cb1efa26739a Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 30 Apr 2025 12:15:37 +0800
Subject: [PATCH 2/9] Fix workflow

---
 .github/workflows/build-perf.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-perf.yaml b/.github/workflows/build-perf.yaml
index 897b41e..24dcd0c 100644
--- a/.github/workflows/build-perf.yaml
+++ b/.github/workflows/build-perf.yaml
@@ -67,10 +67,10 @@ jobs:
       shell: bash
       run: |
         curl -sL https://raw.githubusercontent.com/oracle/coherence-cli/main/scripts/install.sh | bash
-        COHERENCE_VERSION=${{ coherenceVersion }} make test-perf
-       
+        COHERENCE_VERSION=${{ matrix.coherenceVersion }} make test-perf
+
     - uses: actions/upload-artifact@v4
       if: failure()
       with:
-        name: test-output-${{ matrix.go-version }}-${{ coherenceVersion }}
+        name: test-output-${{ matrix.go-version }}-${{ matrix.coherenceVersion }}
         path: build/_output/test-logs
\ No newline at end of file

From c2162fb5bcc3ba28f17ecc7f153ef7a8d4660fe3 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 30 Apr 2025 15:13:15 +0800
Subject: [PATCH 3/9] Updates to perf tests

---
 .../build-compatability-v1-1412.yaml          |   2 +-
 test/e2e/perf/run_test.go                     | 143 +++++++++++++++++-
 test/e2e/perf/suite_test.go                   | 131 ++++++++++++++--
 3 files changed, 258 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/build-compatability-v1-1412.yaml b/.github/workflows/build-compatability-v1-1412.yaml
index cb03e4c..4474e0e 100644
--- a/.github/workflows/build-compatability-v1-1412.yaml
+++ b/.github/workflows/build-compatability-v1-1412.yaml
@@ -23,7 +23,7 @@ jobs:
       fail-fast: false
       matrix:
         coherenceVersion:
-          - 14.1.2-0-1
+          - 14.1.2-0-2
           - 14.1.2-0-2-SNAPSHOT
         go-version:
           - 1.23.x
diff --git a/test/e2e/perf/run_test.go b/test/e2e/perf/run_test.go
index 7cf7ce0..e02a65a 100644
--- a/test/e2e/perf/run_test.go
+++ b/test/e2e/perf/run_test.go
@@ -8,16 +8,151 @@ package perf
 
 import (
 	"github.com/onsi/gomega"
+	"github.com/oracle/coherence-go-client/v2/coherence"
+	"github.com/oracle/coherence-go-client/v2/coherence/aggregators"
+	"github.com/oracle/coherence-go-client/v2/coherence/filters"
 	"testing"
 )
 
-// TestPerformance1 tests for discovery
-func TestPerformance1(t *testing.T) {
+var (
+	filterIDBetween = filters.Between(idExtractor, 125_000, 150_000)
+	filterCountryAU = filters.Equal(countryExtractor, "Australia")
+	filterMultiple  = filters.Equal(countryExtractor, "Australia").And(filterIDBetween)
+	countryInFilter = filters.In(countryExtractor, []string{"Mexico", "Australia"})
+)
+
+// TestStreamingPerformance tests streaming.
+func TestStreamingPerformance(t *testing.T) {
+	var iterations int64 = 10
+
+	testCases := []struct {
+		testName string
+		test     func(t *testing.T, filter filters.Filter, iterations int64) *PerformanceResult
+		filter   filters.Filter
+		count    int64
+	}{
+		{"FilterBetween", RunTestFilterEntrySet, filterIDBetween, iterations},
+		{"FilterEquals", RunTestFilterEntrySet, filterCountryAU, iterations},
+		{"FilterMultiple", RunTestFilterEntrySet, filterMultiple, iterations},
+		{"FilterAlways", RunTestFilterEntrySet, filters.Always(), iterations},
+		{"FilterIn", RunTestFilterEntrySet, countryInFilter, iterations},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.testName, func(t *testing.T) {
+			result := tc.test(t, tc.filter, tc.count)
+			mapResults[tc.testName] = result
+		})
+	}
+}
+
+// TestStreamingPerformance tests streaming.
+func TestAggregatorPerformance(t *testing.T) {
+	var iterations int64 = 10
+
+	testCases := []struct {
+		testName string
+		test     func(t *testing.T, filter filters.Filter, iterations int64) *PerformanceResult
+		filter   filters.Filter
+		count    int64
+	}{
+		{"AggregatorCountAllFilter", RunCountAggregationTest, nil, iterations},
+		{"AggregatorCountCountryFilter", RunCountAggregationTest, filterCountryAU, iterations},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.testName, func(t *testing.T) {
+			result := tc.test(t, tc.filter, tc.count)
+			mapResults[tc.testName] = result
+		})
+	}
+}
+
+// TestKeyOperators tests key operations.
+func TestKeyOperators(t *testing.T) {
+	var iterations int64 = 100_000
+
+	testCases := []struct {
+		testName  string
+		test      func(t *testing.T, operation string, iterations int64) *PerformanceResult
+		operation string
+		count     int64
+	}{
+		{"KeyGet", RunTestKeyOperation, "get", iterations},
+		{"KeyPut", RunTestKeyOperation, "put", iterations},
+		{"KeyContainsKey", RunTestKeyOperation, "containsKey", iterations},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.testName, func(t *testing.T) {
+			result := tc.test(t, tc.operation, tc.count)
+			mapResults[tc.testName] = result
+		})
+	}
+}
+
+// RunTestFilterEntrySet runs tests against various filters.
+func RunTestFilterEntrySet(t *testing.T, filter filters.Filter, count int64) *PerformanceResult {
 	var (
 		g = gomega.NewWithT(t)
+		i int64
 	)
 
+	timer := newTestTimer(count)
+	for i = 0; i < count; i++ {
+		timer.Start()
+		for ch := range config.Students.EntrySetFilter(ctx, filter) {
+			g.Expect(ch.Err).To(gomega.BeNil())
+			_ = ch.Value
+		}
+		timer.End()
+	}
+
+	return timer.Complete()
+}
+
+// RunCountAggregationTest runs tests against various aggregators.
+func RunCountAggregationTest(t *testing.T, filter filters.Filter, count int64) *PerformanceResult {
+	var (
+		g    = gomega.NewWithT(t)
+		i    int64
+		fltr = filters.Always()
+	)
+
+	if filter != nil {
+		fltr = filter
+	}
+
+	timer := newTestTimer(count)
+	for i = 0; i < count; i++ {
+		timer.Start()
+		_, err := coherence.AggregateFilter[int, Student](ctx, config.Students, fltr, aggregators.Count())
+		g.Expect(err).To(gomega.BeNil())
+		timer.End()
+	}
+
+	return timer.Complete()
+}
+
+// RunTestKeyOperation runs key based tests.
+func RunTestKeyOperation(t *testing.T, operation string, count int64) *PerformanceResult {
+	g := gomega.NewWithT(t)
+
 	size, err := config.Students.Size(ctx)
-	g.Expect(err).To(gomega.BeNil())
-	g.Expect(size).To(gomega.Equal(maxStudents))
+	g.Expect(err).NotTo(gomega.HaveOccurred())
+
+	timer := newTestTimer(count)
+	for i := 0; i < int(count); i++ {
+		id := rnd.Intn(size) + 1
+		timer.Start()
+		if operation == "get" {
+			_, err = config.Students.Get(ctx, id)
+		} else if operation == "put" {
+			_, err = config.Students.Put(ctx, id, getRandomStudent(id))
+		} else if operation == "containsKey" {
+			_, err = config.Students.ContainsKey(ctx, id)
+		}
+
+		g.Expect(err).To(gomega.BeNil())
+		timer.End()
+	}
+
+	return timer.Complete()
 }
diff --git a/test/e2e/perf/suite_test.go b/test/e2e/perf/suite_test.go
index f4e7e1c..16d15b3 100644
--- a/test/e2e/perf/suite_test.go
+++ b/test/e2e/perf/suite_test.go
@@ -10,9 +10,11 @@ import (
 	"context"
 	"fmt"
 	"github.com/oracle/coherence-go-client/v2/coherence"
+	"github.com/oracle/coherence-go-client/v2/coherence/extractors"
 	"log"
 	"math/rand"
 	"os"
+	"sort"
 	"testing"
 	"time"
 )
@@ -30,11 +32,23 @@ type Config struct {
 	Students coherence.NamedCache[int, Student]
 }
 
+type PerformanceResult struct {
+	Executions int64
+	TotalTime  int64
+	MinTime    time.Duration
+	MaxTime    time.Duration
+}
+
 var (
 	ctx = context.Background()
-	//nolint:gosec // just a test
-	rnd    = rand.New(rand.NewSource(time.Now().UnixNano()))
+	//nolint:gosec // just a test - have something consistent
+	rnd    = rand.New(rand.NewSource(123_456_789))
 	config = Config{}
+
+	courseExtractor  = extractors.Extract[string]("course")
+	countryExtractor = extractors.Extract[string]("country")
+	idExtractor      = extractors.Extract[int]("id")
+	mapResults       = make(map[string]*PerformanceResult)
 )
 
 const (
@@ -59,24 +73,66 @@ func TestMain(m *testing.M) {
 		errorAndExit("unable to clear", err)
 	}
 
+	// Add Indexes
+	err = coherence.AddIndex[int, Student](ctx, config.Students, courseExtractor, true)
+	if err != nil {
+		errorAndExit("unable to add index", err)
+	}
+
+	err = coherence.AddIndex[int, Student](ctx, config.Students, countryExtractor, true)
+	if err != nil {
+		errorAndExit("unable to add index", err)
+	}
+
+	err = coherence.AddIndex[int, Student](ctx, config.Students, idExtractor, true)
+	if err != nil {
+		errorAndExit("unable to add index", err)
+	}
+
 	log.Println("Populating cache with", maxStudents, "students")
 	if populateCache(config.Students, maxStudents) != nil {
 		errorAndExit("failed to populate cache", err)
 	}
 
 	size, err = config.Students.Size(ctx)
-	if err != nil || size != maxStudents {
-		errorAndExit("size not correct", err)
+	if err != nil {
+		errorAndExit("error", err)
+	}
+	if size != maxStudents {
+		errorAndExit("count", fmt.Errorf("invalid number of students: %d", size))
 	}
 	log.Println("Cache size is", size)
 
 	exitCode := m.Run()
 
 	fmt.Printf("Tests completed with return code %d\n", exitCode)
+	printResults()
 
 	os.Exit(exitCode)
 }
 
+func printResults() {
+	keys := make([]string, 0, len(mapResults))
+	for k := range mapResults {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	// Sort keys
+	sort.Strings(keys)
+	fmt.Println()
+	fmt.Println("RESULTS START")
+	fmt.Println()
+	fmt.Printf("%-30s %15s %15s %15s %15s %15s\n", "TEST", "TOTAL TIME", "EXECUTIONS", "MIN", "MAX", "AVERAGE")
+	for _, k := range keys {
+		v := mapResults[k]
+		fmt.Printf("%-30s %15v %15d %15v %15v %15v\n", k, time.Duration(v.TotalTime), v.Executions,
+			v.MinTime, v.MaxTime, time.Duration(v.TotalTime/v.Executions))
+	}
+	fmt.Println()
+	fmt.Println("RESULTS END")
+}
+
 func errorAndExit(message string, err error) {
 	fmt.Println(message, err)
 	os.Exit(1)
@@ -101,23 +157,20 @@ func InitializeCoherence(ctx context.Context, address string) (Config, error) {
 	return config, nil
 }
 
+var (
+	courses   = []string{"C1", "C2", "C3", "C4"}
+	countries = []string{"Australia", "USA", "Canada", "Mexico"}
+)
+
 func populateCache(cache coherence.NamedMap[int, Student], count int) error {
 	var (
 		buffer    = make(map[int]Student)
 		err       error
-		courses   = []string{"C1", "C2", "C3", "C4"}
-		countries = []string{"Australia", "USA", "Canada", "Mexico"}
 		batchSize = 10_000
 	)
 
 	for i := 1; i <= count; i++ {
-		buffer[i] = Student{
-			ID:      i,
-			Name:    fmt.Sprintf("student%d", i),
-			Address: fmt.Sprintf("address%d", i),
-			Country: randomize(countries),
-			Course:  randomize(courses),
-		}
+		buffer[i] = getRandomStudent(i)
 		if i%batchSize == 0 {
 			err = cache.PutAll(ctx, buffer)
 			if err != nil {
@@ -133,9 +186,61 @@ func populateCache(cache coherence.NamedMap[int, Student], count int) error {
 	return nil
 }
 
+func getRandomStudent(i int) Student {
+	return Student{
+		ID:      i,
+		Name:    fmt.Sprintf("student%d", i),
+		Address: fmt.Sprintf("address%d", i),
+		Country: randomize(countries),
+		Course:  randomize(courses),
+	}
+}
+
 func randomize(arr []string) string {
 	if len(arr) == 0 {
 		return ""
 	}
 	return arr[rnd.Intn(len(arr))]
 }
+
+type testTimer struct {
+	startTime    time.Time
+	endTime      time.Time
+	minDuration  time.Duration
+	maxDuration  time.Duration
+	currentStart time.Time
+	count        int64
+}
+
+func (t *testTimer) Start() {
+	t.currentStart = time.Now()
+}
+
+func (t *testTimer) End() {
+	duration := time.Since(t.currentStart)
+	if duration < t.minDuration {
+		t.minDuration = duration * time.Nanosecond
+	}
+	if duration > t.maxDuration {
+		t.maxDuration = duration * time.Nanosecond
+	}
+}
+
+func (t *testTimer) Complete() *PerformanceResult {
+	t.endTime = time.Now()
+	return &PerformanceResult{
+		Executions: t.count,
+		TotalTime:  t.endTime.Sub(t.startTime).Nanoseconds(),
+		MaxTime:    t.maxDuration,
+		MinTime:    t.minDuration,
+	}
+}
+
+func newTestTimer(count int64) *testTimer {
+	return &testTimer{
+		startTime:   time.Now(),
+		count:       count,
+		minDuration: time.Duration(10000) * time.Second,
+		maxDuration: 0,
+	}
+}

From 013ffa21dccc366509ec46d7204880469e1dec92 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 30 Apr 2025 15:27:44 +0800
Subject: [PATCH 4/9] Configure timeout

---
 .github/workflows/build-perf.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-perf.yaml b/.github/workflows/build-perf.yaml
index 24dcd0c..28f5226 100644
--- a/.github/workflows/build-perf.yaml
+++ b/.github/workflows/build-perf.yaml
@@ -67,7 +67,7 @@ jobs:
       shell: bash
       run: |
         curl -sL https://raw.githubusercontent.com/oracle/coherence-cli/main/scripts/install.sh | bash
-        COHERENCE_VERSION=${{ matrix.coherenceVersion }} make test-perf
+        COHERENCE_CLIENT_REQUEST_TIMEOUT=200000 COHERENCE_VERSION=${{ matrix.coherenceVersion }} make test-perf
 
     - uses: actions/upload-artifact@v4
       if: failure()

From 30d219dd3e17c76fc64b51eb8b884eb37024e6f0 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 30 Apr 2025 15:37:59 +0800
Subject: [PATCH 5/9] Reduce cache size

---
 test/e2e/perf/suite_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/perf/suite_test.go b/test/e2e/perf/suite_test.go
index 16d15b3..caa606d 100644
--- a/test/e2e/perf/suite_test.go
+++ b/test/e2e/perf/suite_test.go
@@ -52,7 +52,7 @@ var (
 )
 
 const (
-	maxStudents = 2_000_000
+	maxStudents = 1_000_000
 )
 
 // The entry point for the test suite

From 8dd71b03a994c299e713da6c70a267131afbfaf6 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 30 Apr 2025 15:47:23 +0800
Subject: [PATCH 6/9] more minor updates to output results

---
 test/e2e/perf/suite_test.go | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/test/e2e/perf/suite_test.go b/test/e2e/perf/suite_test.go
index caa606d..9392235 100644
--- a/test/e2e/perf/suite_test.go
+++ b/test/e2e/perf/suite_test.go
@@ -52,7 +52,7 @@ var (
 )
 
 const (
-	maxStudents = 1_000_000
+	maxStudents = 2_000_000
 )
 
 // The entry point for the test suite
@@ -120,17 +120,17 @@ func printResults() {
 
 	// Sort keys
 	sort.Strings(keys)
-	fmt.Println()
-	fmt.Println("RESULTS START")
-	fmt.Println()
-	fmt.Printf("%-30s %15s %15s %15s %15s %15s\n", "TEST", "TOTAL TIME", "EXECUTIONS", "MIN", "MAX", "AVERAGE")
+	log.Println()
+	log.Println("RESULTS START")
+	log.Println()
+	log.Printf("%-30s %15s %15s %15s %15s %15s\n", "TEST", "TOTAL TIME", "EXECUTIONS", "MIN", "MAX", "AVERAGE")
 	for _, k := range keys {
 		v := mapResults[k]
-		fmt.Printf("%-30s %15v %15d %15v %15v %15v\n", k, time.Duration(v.TotalTime), v.Executions,
+		log.Printf("%-30s %15v %15d %15v %15v %15v\n", k, time.Duration(v.TotalTime), v.Executions,
 			v.MinTime, v.MaxTime, time.Duration(v.TotalTime/v.Executions))
 	}
-	fmt.Println()
-	fmt.Println("RESULTS END")
+	log.Println()
+	log.Println("RESULTS END")
 }
 
 func errorAndExit(message string, err error) {

From 0f6fd4e3a30761901a2c65b0f0ec9296a912f9fe Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 30 Apr 2025 16:07:03 +0800
Subject: [PATCH 7/9] output results

---
 test/e2e/perf/suite_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/e2e/perf/suite_test.go b/test/e2e/perf/suite_test.go
index 9392235..ec66c9b 100644
--- a/test/e2e/perf/suite_test.go
+++ b/test/e2e/perf/suite_test.go
@@ -106,6 +106,8 @@ func TestMain(m *testing.M) {
 	exitCode := m.Run()
 
 	fmt.Printf("Tests completed with return code %d\n", exitCode)
+
+	log.SetOutput(os.Stdout)
 	printResults()
 
 	os.Exit(exitCode)

From 026b38af8421fecd6978f7fb62ab4c8004305fd6 Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 30 Apr 2025 16:37:41 +0800
Subject: [PATCH 8/9] Output to file

---
 .github/workflows/build-perf.yaml |  7 ++++++-
 test/e2e/perf/suite_test.go       | 31 ++++++++++++++++++++-----------
 2 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/build-perf.yaml b/.github/workflows/build-perf.yaml
index 28f5226..f804c78 100644
--- a/.github/workflows/build-perf.yaml
+++ b/.github/workflows/build-perf.yaml
@@ -73,4 +73,9 @@ jobs:
       if: failure()
       with:
         name: test-output-${{ matrix.go-version }}-${{ matrix.coherenceVersion }}
-        path: build/_output/test-logs
\ No newline at end of file
+        path: build/_output/test-logs
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: test-result-${{ matrix.go-version }}-${{ matrix.coherenceVersion }}
+        path:  test/e2e/perf/results.txt
diff --git a/test/e2e/perf/suite_test.go b/test/e2e/perf/suite_test.go
index ec66c9b..4ba3e3c 100644
--- a/test/e2e/perf/suite_test.go
+++ b/test/e2e/perf/suite_test.go
@@ -15,6 +15,7 @@ import (
 	"math/rand"
 	"os"
 	"sort"
+	"strings"
 	"testing"
 	"time"
 )
@@ -107,7 +108,6 @@ func TestMain(m *testing.M) {
 
 	fmt.Printf("Tests completed with return code %d\n", exitCode)
 
-	log.SetOutput(os.Stdout)
 	printResults()
 
 	os.Exit(exitCode)
@@ -120,19 +120,28 @@ func printResults() {
 	}
 	sort.Strings(keys)
 
-	// Sort keys
-	sort.Strings(keys)
-	log.Println()
-	log.Println("RESULTS START")
-	log.Println()
-	log.Printf("%-30s %15s %15s %15s %15s %15s\n", "TEST", "TOTAL TIME", "EXECUTIONS", "MIN", "MAX", "AVERAGE")
+	file, err := os.Create("results.txt")
+	if err != nil {
+		panic(err)
+	}
+	defer file.Close()
+
+	var sb strings.Builder
+
+	sb.WriteString("\n")
+	sb.WriteString("RESULTS START")
+	sb.WriteString("\n")
+	sb.WriteString(fmt.Sprintf("%-30s %15s %15s %15s %15s %15s\n", "TEST", "TOTAL TIME", "EXECUTIONS", "MIN", "MAX", "AVERAGE"))
 	for _, k := range keys {
 		v := mapResults[k]
-		log.Printf("%-30s %15v %15d %15v %15v %15v\n", k, time.Duration(v.TotalTime), v.Executions,
-			v.MinTime, v.MaxTime, time.Duration(v.TotalTime/v.Executions))
+		sb.WriteString(fmt.Sprintf("%-30s %15v %15d %15v %15v %15v\n", k, time.Duration(v.TotalTime), v.Executions,
+			v.MinTime, v.MaxTime, time.Duration(v.TotalTime/v.Executions)))
 	}
-	log.Println()
-	log.Println("RESULTS END")
+	sb.WriteString("\n")
+	sb.WriteString("RESULTS END\n")
+
+	_, err = fmt.Fprintln(file, sb.String())
+
 }
 
 func errorAndExit(message string, err error) {

From 9b9d66ad9b37c38d74638bf608eb60be600747fc Mon Sep 17 00:00:00 2001
From: Tim Middleton <tim.middleton@oracle.com>
Date: Wed, 30 Apr 2025 16:54:31 +0800
Subject: [PATCH 9/9] Minor

---
 test/e2e/perf/suite_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/e2e/perf/suite_test.go b/test/e2e/perf/suite_test.go
index 4ba3e3c..27944cd 100644
--- a/test/e2e/perf/suite_test.go
+++ b/test/e2e/perf/suite_test.go
@@ -140,8 +140,8 @@ func printResults() {
 	sb.WriteString("\n")
 	sb.WriteString("RESULTS END\n")
 
-	_, err = fmt.Fprintln(file, sb.String())
-
+	_, _ = fmt.Fprintln(file, sb.String())
+	_, _ = fmt.Fprintln(file, sb.String())
 }
 
 func errorAndExit(message string, err error) {