[automatic failover] Implement ping health check (CAE-1687) (#3564)

* add Ping strategy

* add PingStrategyIntegrationTests

add integration test

* health checks refactored (inject DatabaseConnectionProvider instead ClientOptions

Inject DatabaseConnectionProvider into HealthCheckStrategySupplier's. Injecting per DB connection factory allows reuse of  MultiDB client resources

  - ClientOptions no longer propagated to HealthCheckStrategySupplier
  - HealthCheckStrategySupplier refactored to use DatabaseConnectionProvider

* clean up

  - renamed DatabaseConnectionProvider -> DatabaseRawConnectionFactory
  - api docs updated

* format

* Fix sporadic test failures

 - Shared TestClientResources shutdown during tests, caused subsequent test to fail.

* clean up - rename internal vars

* clean up

   - add unit test
   - remove unused HealthCheckStrategySupplier DEFAULT_WITH_PROVIDER

Author: ggivo

src/main/java/io/lettuce/core/failover/DatabaseRawConnectionFactory.java

@@ -0,0 +1,25 @@
+package io.lettuce.core.failover;
+
+import io.lettuce.core.RedisURI;
+import io.lettuce.core.api.StatefulRedisConnection;
+
+/**
+ * Factory interface to obtain direct {@link StatefulRedisConnection connections} to Redis database nodes. Connections created
+ * by this factory are raw connections without CircuitBreaker counting, health checks, or other management features.
+ *
+ * @author Ivo Gaydazhiev
+ * @since 7.2
+ */
+@FunctionalInterface
+public interface DatabaseRawConnectionFactory {
+
+    /**
+     * Creates a new bare connection to the specified database endpoint. The connection is created without CircuitBreaker
+     * counting, health checks, or other management features.
+     *
+     * @param endpoint the Redis URI of the database endpoint
+     * @return a new stateful Redis connection to the specified endpoint
+     */
+    StatefulRedisConnection<?, ?> connectToDatabase(RedisURI endpoint);
+
+}

src/main/java/io/lettuce/core/failover/MultiDbClientImpl.java

@@ -47,6 +47,8 @@ class MultiDbClientImpl extends RedisClient implements MultiDbClient {
 
     private final Map<RedisURI, DatabaseConfig> databaseConfigs;
 
+    private final DatabaseRawConnectionFactory databaseRawConnectionFactory = new DatabaseRawConnectionFactoryImpl();
+
     MultiDbClientImpl(Collection<DatabaseConfig> databaseConfigs) {
         this(null, databaseConfigs);
     }
@@ -125,7 +127,7 @@ private <K, V> RedisDatabase<StatefulRedisConnection<K, V>> createRedisDatabase(
         HealthCheck healthCheck;
         if (config.getHealthCheckStrategySupplier() != null) {
             HealthCheckStrategy hcStrategy = config.getHealthCheckStrategySupplier().get(config.getRedisURI(),
-                    connection.getOptions());
+                    databaseRawConnectionFactory);
             healthCheck = healthStatusManager.add(uri, hcStrategy);
         } else {
             healthCheck = null;
@@ -184,8 +186,9 @@ private <K, V> RedisDatabase<StatefulRedisPubSubConnection<K, V>> createRedisDat
 
         HealthCheck healthCheck;
         if (config.getHealthCheckStrategySupplier() != null) {
+
             HealthCheckStrategy hcStrategy = config.getHealthCheckStrategySupplier().get(config.getRedisURI(),
-                    connection.getOptions());
+                    databaseRawConnectionFactory);
             healthCheck = healthStatusManager.add(uri, hcStrategy);
         } else {
             healthCheck = null;
@@ -262,4 +265,13 @@ private void waitForInitialHealthyDatabase(StatusTracker statusTracker,
         throw new RedisConnectionException("All configured databases are unhealthy.");
     }
 
+    private class DatabaseRawConnectionFactoryImpl implements DatabaseRawConnectionFactory {
+
+        @Override
+        public StatefulRedisConnection<?, ?> connectToDatabase(RedisURI endpoint) {
+            return MultiDbClientImpl.this.connect(endpoint);
+        }
+
+    }
+
 }

src/main/java/io/lettuce/core/failover/health/HealthCheckStrategySupplier.java

@@ -1,7 +1,7 @@
 package io.lettuce.core.failover.health;
 
-import io.lettuce.core.ClientOptions;
 import io.lettuce.core.RedisURI;
+import io.lettuce.core.failover.DatabaseRawConnectionFactory;
 
 /**
  * Supplier for health check strategies.
@@ -18,6 +18,6 @@ public interface HealthCheckStrategySupplier {
      * @param clientOptions the client options
      * @return the health check strategy
      */
-    HealthCheckStrategy get(RedisURI redisURI, ClientOptions clientOptions);
+    HealthCheckStrategy get(RedisURI redisURI, DatabaseRawConnectionFactory connectionProvider);
 
 }

src/main/java/io/lettuce/core/failover/health/PingStrategy.java

@@ -0,0 +1,68 @@
+package io.lettuce.core.failover.health;
+
+import io.lettuce.core.RedisURI;
+import io.lettuce.core.api.StatefulRedisConnection;
+import io.lettuce.core.failover.DatabaseRawConnectionFactory;
+
+public class PingStrategy implements HealthCheckStrategy {
+
+    private final DatabaseRawConnectionFactory connectionFactory;
+
+    private final HealthCheckStrategy.Config config;
+
+    public PingStrategy(RedisURI redisURI, DatabaseRawConnectionFactory connectionFactory) {
+        this(redisURI, connectionFactory, HealthCheckStrategy.Config.create());
+    }
+
+    public PingStrategy(RedisURI redisURI, DatabaseRawConnectionFactory connectionFactory, HealthCheckStrategy.Config config) {
+        this.connectionFactory = connectionFactory;
+        this.config = config;
+    }
+
+    @Override
+    public int getInterval() {
+        return config.getInterval();
+    }
+
+    @Override
+    public int getTimeout() {
+        return config.getTimeout();
+    }
+
+    @Override
+    public int getNumProbes() {
+        return config.getNumProbes();
+    }
+
+    @Override
+    public ProbingPolicy getPolicy() {
+        return config.getPolicy();
+    }
+
+    @Override
+    public int getDelayInBetweenProbes() {
+        return config.getDelayInBetweenProbes();
+    }
+
+    @Override
+    public HealthStatus doHealthCheck(RedisURI endpoint) {
+        try (StatefulRedisConnection<?, ?> connection = connectionFactory.connectToDatabase(endpoint)) {
+
+            if (connection == null) {
+                return HealthStatus.UNHEALTHY;
+            }
+
+            return "PONG".equals(connection.sync().ping()) ? HealthStatus.HEALTHY : HealthStatus.UNHEALTHY;
+        } catch (Exception e) {
+            return HealthStatus.UNHEALTHY;
+        }
+    }
+
+    @Override
+    public void close() {
+        // No resources to close
+    }
+
+    public static final HealthCheckStrategySupplier DEFAULT = PingStrategy::new;
+
+}

src/test/java/io/lettuce/core/failover/DatabaseEndpointCallbackTests.java

@@ -54,9 +54,6 @@ void setUp() {
 
     @AfterEach
     void tearDown() {
-        if (clientResources != null) {
-            FastShutdown.shutdown(clientResources);
-        }
         if (endpoint != null) {
             endpoint.close();
         }