Merge branch 'main' into ccs-many-it

davidkyle · web-flow · commit ecee4718e197 · 2025-12-08T15:33:11.000Z
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
@@ -182,7 +182,7 @@ public List<RestHandler> getRestHandlers(
         Predicate<NodeFeature> clusterSupportsFeature
     ) {
         List<RestHandler> handlers = new ArrayList<>();
-        handlers.add(new PainlessExecuteAction.RestAction());
+        handlers.add(new PainlessExecuteAction.RestAction(settings));
         handlers.add(new PainlessContextAction.RestAction());
         return handlers;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java
@@ -47,6 +47,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.network.NetworkAddress;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -94,6 +95,7 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.script.StringFieldScript;
+import org.elasticsearch.search.crossproject.CrossProjectModeDecider;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -351,6 +353,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         private final Script script;
         private final ScriptContext<?> context;
         private final ContextSetup contextSetup;
+        private transient IndicesOptions indicesOptions;
 
         static Request parse(XContentParser parser) throws IOException {
             return PARSER.parse(parser, null);
@@ -390,6 +393,14 @@ public ContextSetup getContextSetup() {
             return contextSetup;
         }
 
+        @Override
+        public void markOriginOnly() {
+            assert contextSetup != null
+                : "Painless/execute request without context setup can't have index, this method shouldn't be called";
+            // strip off cluster alias from the index in this request
+            index(contextSetup.getIndex());
+        }
+
         @Override
         public ActionRequestValidationException validate() {
             ActionRequestValidationException validationException = null;
@@ -407,6 +418,20 @@ public ActionRequestValidationException validate() {
             return validationException;
         }
 
+        public void enableCrossProjectMode() {
+            this.indicesOptions = IndicesOptions.builder(super.indicesOptions())
+                .crossProjectModeOptions(new IndicesOptions.CrossProjectModeOptions(true))
+                .build();
+        }
+
+        @Override
+        public IndicesOptions indicesOptions() {
+            if (indicesOptions == null) {
+                return super.indicesOptions();
+            }
+            return indicesOptions;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
@@ -532,7 +557,11 @@ public TransportAction(
 
         @Override
         protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
-            if (request.getContextSetup() == null || request.getContextSetup().getClusterAlias() == null) {
+            // By this point index resolution has completed, and we should not try to resolve indices for child requests
+            // to avoid the second attempt of project authorization in CPS
+            request.indicesOptions = null;
+
+            if (isLocalIndex(request)) {
                 super.doExecute(task, request, listener);
             } else {
                 // forward to remote cluster after stripping off the clusterAlias from the index expression
@@ -547,10 +576,19 @@ protected void doExecute(Task task, Request request, ActionListener<Response> li
             }
         }
 
+        private boolean isLocalIndex(Request request) {
+            if (request.getContextSetup() == null) {
+                return true;
+            }
+            String index = request.index();
+            return index == null || RemoteClusterAware.isRemoteIndexName(index) == false;
+        }
+
         // Visible for testing
         static void removeClusterAliasFromIndexExpression(Request request) {
-            if (request.index() != null) {
-                String[] split = RemoteClusterAware.splitIndexName(request.index());
+            String index = request.index();
+            if (index != null) {
+                String[] split = RemoteClusterAware.splitIndexName(index);
                 if (split[0] != null) {
                     /*
                      * if the cluster alias is null and the index field has a clusterAlias (clusterAlias:index notation)
@@ -854,6 +892,11 @@ private static Response prepareRamIndex(
 
     @ServerlessScope(Scope.PUBLIC)
     public static class RestAction extends BaseRestHandler {
+        private final CrossProjectModeDecider crossProjectModeDecider;
+
+        public RestAction(Settings settings) {
+            this.crossProjectModeDecider = new CrossProjectModeDecider(settings);
+        }
 
         @Override
         public List<Route> routes() {
@@ -868,6 +911,10 @@ public String getName() {
         @Override
         protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
             final Request request = Request.parse(restRequest.contentOrSourceParamParser());
+            if (crossProjectModeDecider.crossProjectEnabled()) {
+                request.enableCrossProjectMode();
+            }
+
             return channel -> client.executeLocally(INSTANCE, request, new RestToXContentListener<>(channel));
         }
     }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteApiTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteApiTests.java
@@ -561,6 +561,12 @@ public void testRemoveClusterAliasFromIndexExpression() {
         }
     }
 
+    public void testMarkOriginOnly() {
+        PainlessExecuteAction.Request request = createRequest("p1:blogs");
+        request.markOriginOnly();
+        assertThat(request.index(), equalTo("blogs"));
+    }
+
     private PainlessExecuteAction.Request createRequest(String indexExpression) {
         return new PainlessExecuteAction.Request(
             new Script("100.0 / 1000.0"),
diff --git a/muted-tests.yml b/muted-tests.yml
@@ -509,6 +509,12 @@ tests:
 - class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
   method: test {p0=eql/10_basic/Sequence checking correct join key ordering.}
   issue: https://github.com/elastic/elasticsearch/issues/139173
+- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
+  method: test {p0=search.retrievers/result-diversification/10_mmr_result_diversification_retriever/Test MMR result diversification single index float type}
+  issue: https://github.com/elastic/elasticsearch/issues/139195
+- class: org.elasticsearch.smoketest.MlWithSecurityIT
+  method: test {yaml=ml/sparse_vector_search/Test sparse_vector search with query vector and pruning config}
+  issue: https://github.com/elastic/elasticsearch/issues/139196
 
 # Examples:
 #
diff --git a/server/src/main/java/org/elasticsearch/action/IndicesRequest.java b/server/src/main/java/org/elasticsearch/action/IndicesRequest.java
@@ -118,6 +118,20 @@ interface SingleIndexNoWildcards extends IndicesRequest, CrossProjectCandidate {
         default boolean allowsRemoteIndices() {
             return true;
         }
+
+        /**
+         * Determines whether the request type allows cross-project processing. Cross-project processing entails cross-project search
+         * index resolution and error handling. Note: this method only determines in the request _supports_ cross-project.
+         * Whether cross-project processing is actually performed is determined by {@link IndicesOptions}.
+         */
+        default boolean allowsCrossProject() {
+            return true;
+        }
+
+        /**
+         * Marks request local. Local requests should be processed on the same cluster, even if they have cluster-alias prefix.
+         */
+        void markOriginOnly();
     }
 
     /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java
@@ -180,7 +180,9 @@ public class InternalUsers {
                         // System data stream for result history of fleet actions (see Fleet#fleetActionsResultsDescriptor)
                         ".fleet-actions-results",
                         // System data streams for storing uploaded file data for Agent diagnostics and Endpoint response actions
-                        ".fleet-fileds*"
+                        ".fleet-fileds*",
+                        // System data stream for kibana workflows logs
+                        ".workflows-execution-data-stream-logs"
                     )
                     .privileges(
                         filterNonNull(
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/InternalUsersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/InternalUsersTests.java
@@ -253,7 +253,11 @@ public void testDataStreamLifecycleUser() {
         assertThat(role.application(), is(ApplicationPermission.NONE));
         assertThat(role.remoteIndices(), is(RemoteIndicesPermission.NONE));
 
-        final List<String> allowedSystemDataStreams = Arrays.asList(".fleet-actions-results", ".fleet-fileds*");
+        final List<String> allowedSystemDataStreams = Arrays.asList(
+            ".fleet-actions-results",
+            ".fleet-fileds*",
+            ".workflows-execution-data-stream-logs"
+        );
         for (var group : role.indices().groups()) {
             if (group.allowRestrictedIndices()) {
                 assertThat(group.indices(), arrayContaining(allowedSystemDataStreams.toArray(new String[0])));
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlRestValidationTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlRestValidationTestCase.java
@@ -8,8 +8,6 @@
 package org.elasticsearch.xpack.esql.qa.rest;
 
 import org.elasticsearch.client.Request;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.WarningsHandler;
@@ -20,6 +18,7 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.List;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -28,24 +27,6 @@ public abstract class EsqlRestValidationTestCase extends ESRestTestCase {
 
     private static final String indexName = "test_esql";
     private static final String aliasName = "alias-test_esql";
-    protected static final String[] existentIndexWithWildcard = new String[] {
-        indexName + ",inexistent*",
-        indexName + "*,inexistent*",
-        "inexistent*," + indexName };
-    private static final String[] existentIndexWithoutWildcard = new String[] { indexName + ",inexistent", "inexistent," + indexName };
-    protected static final String[] existentAliasWithWildcard = new String[] {
-        aliasName + ",inexistent*",
-        aliasName + "*,inexistent*",
-        "inexistent*," + aliasName };
-    private static final String[] existentAliasWithoutWildcard = new String[] { aliasName + ",inexistent", "inexistent," + aliasName };
-    private static final String[] inexistentIndexNameWithWildcard = new String[] { "inexistent*", "inexistent1*,inexistent2*" };
-    private static final String[] inexistentIndexNameWithoutWildcard = new String[] { "inexistent", "inexistent1,inexistent2" };
-    private static final String createAlias = "{\"actions\":[{\"add\":{\"index\":\"" + indexName + "\",\"alias\":\"" + aliasName + "\"}}]}";
-    private static final String removeAlias = "{\"actions\":[{\"remove\":{\"index\":\""
-        + indexName
-        + "\",\"alias\":\""
-        + aliasName
-        + "\"}}]}";
 
     @Before
     @After
@@ -73,79 +54,76 @@ public void wipeTestData() throws IOException {
         }
     }
 
-    private String getInexistentIndexErrorMessage() {
-        return "\"reason\" : \"Found 1 problem\\nline 1:1: Unknown index ";
-    }
-
-    public void testInexistentIndexNameWithWildcard() throws IOException {
-        assertErrorMessages(inexistentIndexNameWithWildcard, getInexistentIndexErrorMessage(), 400);
+    public void testInexistentIndexNameWithWildcard() {
+        for (String pattern : List.of("inexistent*", "inexistent1*,inexistent2*")) {
+            assertError(pattern, 400, "Found 1 problem\\nline 1:1: Unknown index [" + clusterSpecificIndexName(pattern) + "]");
+        }
     }
 
-    public void testInexistentIndexNameWithoutWildcard() throws IOException {
-        assertErrorMessages(inexistentIndexNameWithoutWildcard, getInexistentIndexErrorMessage(), 400);
+    public void testInexistentIndexNameWithoutWildcard() {
+        for (String pattern : List.of("inexistent", "inexistent1,inexistent2")) {
+            assertError(pattern, "Found 1 problem\\nline 1:1: Unknown index [" + clusterSpecificIndexName(pattern) + "]", 400);
+        }
     }
 
     public void testExistentIndexWithoutWildcard() throws IOException {
-        for (String indexName : existentIndexWithoutWildcard) {
-            assertErrorMessage(indexName, "\"reason\" : \"no such index [inexistent]\"", 404);
+        for (String pattern : List.of(indexName + ",inexistent", "inexistent," + indexName)) {
+            assertError(pattern, 404, "no such index [inexistent]");
         }
     }
 
     public void testExistentIndexWithWildcard() throws IOException {
-        assertValidRequestOnIndices(existentIndexWithWildcard);
+        for (String pattern : List.of(indexName + ",inexistent*", indexName + "*,inexistent*", "inexistent*," + indexName)) {
+            assertOK(client().performRequest(createRequest(pattern)));
+        }
     }
 
     public void testAlias() throws IOException {
-        createAlias();
+        updateAliases("""
+            {"actions":[{"add":{"index":"%s","alias":"%s"}}]}
+            """.formatted(indexName, aliasName));
 
-        for (String indexName : existentAliasWithoutWildcard) {
-            assertErrorMessage(indexName, "\"reason\" : \"no such index [inexistent]\"", 404);
+        for (String indexName : List.of(aliasName + ",inexistent", "inexistent," + aliasName)) {
+            assertError(indexName, "no such index [inexistent]", 404);
         }
-        assertValidRequestOnIndices(existentAliasWithWildcard);
-
-        deleteAlias();
-    }
-
-    private void assertErrorMessages(String[] indices, String errorMessage, int statusCode) throws IOException {
-        for (String indexName : indices) {
-            assertErrorMessage(indexName, errorMessage + "[" + clusterSpecificIndexName(indexName) + "]", statusCode);
+        for (String indexName : List.of(aliasName + ",inexistent*", aliasName + "*,inexistent*", "inexistent*," + aliasName)) {
+            assertOK(client().performRequest(createRequest(indexName)));
         }
+
+        updateAliases("""
+            {"actions":[{"remove":{"index":"%s","alias":"%s"}}]}
+            """.formatted(indexName, aliasName));
     }
 
     protected String clusterSpecificIndexName(String indexName) {
         return indexName;
     }
 
-    private void assertErrorMessage(String indexName, String errorMessage, int statusCode) throws IOException {
-        var specificName = clusterSpecificIndexName(indexName);
-        final var request = createRequest(specificName);
-        ResponseException exc = expectThrows(ResponseException.class, () -> client().performRequest(request));
+    private void assertError(String indexName, String errorMessage, int statusCode) {
+        ResponseException exc = expectThrows(ResponseException.class, () -> client().performRequest(createRequest(indexName)));
+        assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(statusCode));
+        assertThat(exc.getMessage(), containsString("\"reason\" : \"" + errorMessage + "\""));
+    }
 
+    private void assertError(String indexName, int statusCode, String errorMessage) {
+        ResponseException exc = expectThrows(ResponseException.class, () -> client().performRequest(createRequest(indexName)));
         assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(statusCode));
-        assertThat(exc.getMessage(), containsString(errorMessage));
+        assertThat(exc.getMessage(), containsString("\"reason\" : \"" + errorMessage + "\""));
     }
 
     private Request createRequest(String indexName) throws IOException {
         final var request = new Request("POST", "/_query");
         request.addParameter("error_trace", "true");
         request.addParameter("pretty", "true");
         request.setJsonEntity(
-            Strings.toString(JsonXContent.contentBuilder().startObject().field("query", "from " + indexName).endObject())
+            Strings.toString(
+                JsonXContent.contentBuilder().startObject().field("query", "from " + clusterSpecificIndexName(indexName)).endObject()
+            )
         );
-        RequestOptions.Builder options = request.getOptions().toBuilder();
-        options.setWarningsHandler(WarningsHandler.PERMISSIVE);
-        request.setOptions(options);
+        request.setOptions(request.getOptions().toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE));
         return request;
     }
 
-    private void assertValidRequestOnIndices(String[] indices) throws IOException {
-        for (String indexName : indices) {
-            final var request = createRequest(clusterSpecificIndexName(indexName));
-            Response response = client().performRequest(request);
-            assertOK(response);
-        }
-    }
-
     // Returned client is used to load the test data, either in the local cluster or a remote one (for
     // multi-clusters). The client()/adminClient() will always connect to the local cluster
     protected RestClient provisioningClient() throws IOException {
@@ -156,15 +134,9 @@ protected RestClient provisioningAdminClient() throws IOException {
         return adminClient();
     }
 
-    private void createAlias() throws IOException {
+    private void updateAliases(String update) throws IOException {
         var r = new Request("POST", "_aliases");
-        r.setJsonEntity(createAlias);
+        r.setJsonEntity(update);
         assertOK(provisioningClient().performRequest(r));
     }
-
-    private void deleteAlias() throws IOException {
-        var r = new Request("POST", "/_aliases/");
-        r.setJsonEntity(removeAlias);
-        assertOK(provisioningAdminClient().performRequest(r));
-    }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
@@ -573,7 +573,7 @@ private AsyncSupplier<ResolvedIndices> makeResolvedIndicesAsyncSupplier(
                 var resolvedIndices = indicesAndAliasesResolver.resolvePITIndices(searchRequest);
                 return SubscribableListener.newSucceeded(resolvedIndices);
             }
-            final ResolvedIndices resolvedIndices = indicesAndAliasesResolver.tryResolveWithoutWildcards(action, request);
+            final ResolvedIndices resolvedIndices = indicesAndAliasesResolver.tryResolveWithoutWildcards(action, request, targetProjects);
             if (resolvedIndices != null) {
                 return SubscribableListener.newSucceeded(resolvedIndices);
             } else {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java

Original file line number	Diff line number	Diff line change
`@@ -182,7 +182,7 @@ public List<RestHandler> getRestHandlers(`
`182`	`182`	`Predicate<NodeFeature> clusterSupportsFeature`
`183`	`183`	`) {`
`184`	`184`	`List<RestHandler> handlers = new ArrayList<>();`
`185`		`- handlers.add(new PainlessExecuteAction.RestAction());`
	`185`	`+ handlers.add(new PainlessExecuteAction.RestAction(settings));`
`186`	`186`	`handlers.add(new PainlessContextAction.RestAction());`
`187`	`187`	`return handlers;`
`188`	`188`	`}`
Original file line number	Diff line number	Diff line change
`@@ -561,6 +561,12 @@ public void testRemoveClusterAliasFromIndexExpression() {`
`561`	`561`	`}`
`562`	`562`	`}`
`563`	`563`
	`564`	`+ public void testMarkOriginOnly() {`
	`565`	`+ PainlessExecuteAction.Request request = createRequest("p1:blogs");`
	`566`	`+ request.markOriginOnly();`
	`567`	`+ assertThat(request.index(), equalTo("blogs"));`
	`568`	`+ }`
	`569`	`+`
`564`	`570`	`private PainlessExecuteAction.Request createRequest(String indexExpression) {`
`565`	`571`	`return new PainlessExecuteAction.Request(`
`566`	`572`	`new Script("100.0 / 1000.0"),`
Original file line number	Diff line number	Diff line change
`@@ -573,7 +573,7 @@ private AsyncSupplier<ResolvedIndices> makeResolvedIndicesAsyncSupplier(`
`573`	`573`	`var resolvedIndices = indicesAndAliasesResolver.resolvePITIndices(searchRequest);`
`574`	`574`	`return SubscribableListener.newSucceeded(resolvedIndices);`
`575`	`575`	`}`
`576`		`- final ResolvedIndices resolvedIndices = indicesAndAliasesResolver.tryResolveWithoutWildcards(action, request);`
	`576`	`+ final ResolvedIndices resolvedIndices = indicesAndAliasesResolver.tryResolveWithoutWildcards(action, request, targetProjects);`
`577`	`577`	`if (resolvedIndices != null) {`
`578`	`578`	`return SubscribableListener.newSucceeded(resolvedIndices);`
`579`	`579`	`} else {`