Ack node shutdown requests after cluster state update is complete (#85846)

The adding or removing a node to the node shutdown api stores submits a
cluster state update task to store the node metadata to be shutdown.
Sometimes a reroute is required, and in those cases we currently wait
for the reroute to complete before acknowledging the original request.
This commit changes the behavior to acknowledge success after the
cluster state update is complete. A warning is still logged if the
reroute fails.

relates #84847

Author: rjernst

x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeAction.java

@@ -93,22 +93,16 @@ public void onFailure(Exception e) {
             @Override
             public void clusterStateProcessed(ClusterState oldState, ClusterState newState) {
                 clusterService.getRerouteService()
-                    .reroute("node registered for removal from cluster", Priority.URGENT, new ActionListener<ClusterState>() {
+                    .reroute("node registered for removal from cluster", Priority.URGENT, new ActionListener<>() {
                         @Override
-                        public void onResponse(ClusterState clusterState) {
-                            logger.trace("started reroute after deleting node [{}}] shutdown", request.getNodeId());
-                            listener.onResponse(AcknowledgedResponse.TRUE);
-                        }
+                        public void onResponse(ClusterState clusterState) {}
 
                         @Override
                         public void onFailure(Exception e) {
-                            logger.warn(
-                                new ParameterizedMessage("failed to start reroute after deleting node [{}] shutdown", request.getNodeId()),
-                                e
-                            );
-                            listener.onFailure(e);
+                            logger.warn(() -> "failed to reroute after deleting node [" + request.getNodeId() + "] shutdown", e);
                         }
                     });
+                listener.onResponse(AcknowledgedResponse.TRUE);
             }
         }, newExecutor());
     }

x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java

@@ -102,38 +102,32 @@ public void onFailure(Exception e) {
 
             @Override
             public void clusterStateProcessed(ClusterState oldState, ClusterState newState) {
-                if (SingleNodeShutdownMetadata.Type.REMOVE.equals(request.getType())
-                    || SingleNodeShutdownMetadata.Type.REPLACE.equals(request.getType())) {
+                boolean shouldReroute = switch (request.getType()) {
+                    case REMOVE, REPLACE -> true;
+                    default -> false;
+                };
+
+                if (shouldReroute) {
                     clusterService.getRerouteService()
-                        .reroute("node registered for removal from cluster", Priority.URGENT, new ActionListener<ClusterState>() {
+                        .reroute("node registered for removal from cluster", Priority.URGENT, new ActionListener<>() {
                             @Override
-                            public void onResponse(ClusterState clusterState) {
-                                logger.trace("started reroute after registering node [{}] for removal", request.getNodeId());
-                                listener.onResponse(AcknowledgedResponse.TRUE);
-                            }
+                            public void onResponse(ClusterState clusterState) {}
 
                             @Override
                             public void onFailure(Exception e) {
-                                logger.warn(
-                                    new ParameterizedMessage(
-                                        "failed to start reroute after registering node [{}] for removal",
-                                        request.getNodeId()
-                                    ),
-                                    e
-                                );
-                                listener.onFailure(e);
+                                logger.warn(() -> "failed to reroute after registering node [" + request.getNodeId() + "] for shutdown", e);
                             }
                         });
                 } else {
                     logger.trace(
-                        "not starting reroute after registering node ["
+                        () -> "not starting reroute after registering node ["
                             + request.getNodeId()
                             + "] for shutdown of type ["
                             + request.getType()
                             + "]"
                     );
-                    listener.onResponse(AcknowledgedResponse.TRUE);
                 }
+                listener.onResponse(AcknowledgedResponse.TRUE);
             }
         }, newExecutor());
     }