From 007a309477e7fd79c8ad07868711d77aa660d52b Mon Sep 17 00:00:00 2001 From: afoucret Date: Tue, 25 Nov 2025 14:58:46 +0100 Subject: [PATCH 01/12] Fix stats after InferencePlan (RERANK and COMPLETION). --- .../src/main/resources/completion.csv-spec | 15 +++++++++++++++ .../src/main/resources/rerank.csv-spec | 15 +++++++++++++++ .../plan/logical/inference/InferencePlan.java | 4 +++- .../xpack/esql/planner/mapper/Mapper.java | 16 ++++------------ 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec index 9e2f88fd99d42..feaa4637c70e1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec @@ -59,3 +59,18 @@ title:text | completion:keyword War and Peace | THIS IS A PROMPT: WAR AND PEACE War and Peace (Signet Classics) | THIS IS A PROMPT: WAR AND PEACE (SIGNET CLASSICS) ; + + +completion followed by stats +required_capability: completion +required_capability: match_operator_colon + +FROM books METADATA _score +| WHERE title:"war and peace" AND author:"Tolstoy" +| COMPLETION CONCAT("This is a prompt: ", title) WITH { "inference_id" : "test_completion" } +| STATS count=COUNT(*), avg_completion_length = AVG(LENGTH(completion)) +; + +count:long | avg_completion_length:double +4 | 50.75 +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec index e7eb7dd3c7021..ef772be88dd3c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec @@ -294,3 +294,18 @@ The Lord of the Rings - Boxed Set | 3.76885509490 Return of the King Being the Third Part of The Lord of the Rings | 3.6248698234558105 | 9.000900317914784E-4 | 0.001396648003719747 // end::combine-result[] ; + +reranker followed by stats, overwrite existing _score column +required_capability: rerank +required_capability: match_operator_colon + +FROM books METADATA _score +| WHERE title:"war and peace" AND author:"Tolstoy" +| SORT _score DESC, book_no ASC +| RERANK "war and peace" ON title WITH { "inference_id" : "test_reranker" } +| STATS count_book = COUNT(*) WHERE _score >= 0.03 +; + +count_book:long +2 +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/inference/InferencePlan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/inference/InferencePlan.java index 06db9cd06405a..9e1bae8e2c44c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/inference/InferencePlan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/inference/InferencePlan.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.plan.GeneratingPlan; import org.elasticsearch.xpack.esql.plan.logical.ExecutesOn; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.PipelineBreaker; import org.elasticsearch.xpack.esql.plan.logical.SortAgnostic; import org.elasticsearch.xpack.esql.plan.logical.Streaming; import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; @@ -28,7 +29,8 @@ public abstract class InferencePlan> ex Streaming, SortAgnostic, GeneratingPlan>, - ExecutesOn.Coordinator { + ExecutesOn.Coordinator, + PipelineBreaker { public static final String INFERENCE_ID_OPTION_NAME = "inference_id"; public static final List VALID_INFERENCE_OPTION_NAMES = List.of(INFERENCE_ID_OPTION_NAME); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java index 95097915c3c45..aff4f7d0c69a9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java @@ -24,7 +24,7 @@ import org.elasticsearch.xpack.esql.plan.logical.PipelineBreaker; import org.elasticsearch.xpack.esql.plan.logical.TopN; import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; -import org.elasticsearch.xpack.esql.plan.logical.inference.Rerank; +import org.elasticsearch.xpack.esql.plan.logical.inference.InferencePlan; import org.elasticsearch.xpack.esql.plan.logical.join.Join; import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig; import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; @@ -37,7 +37,6 @@ import org.elasticsearch.xpack.esql.plan.physical.MergeExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.plan.physical.TopNExec; -import org.elasticsearch.xpack.esql.plan.physical.inference.RerankExec; import org.elasticsearch.xpack.esql.session.Versioned; import java.util.List; @@ -141,16 +140,9 @@ else if (aggregate.groupings() return new TopNExec(topN.source(), mappedChild, topN.order(), topN.limit(), null); } - if (unary instanceof Rerank rerank) { - mappedChild = addExchangeForFragment(rerank, mappedChild); - return new RerankExec( - rerank.source(), - mappedChild, - rerank.inferenceId(), - rerank.queryText(), - rerank.rerankFields(), - rerank.scoreAttribute() - ); + if (unary instanceof InferencePlan inferencePlan) { + mappedChild = addExchangeForFragment(inferencePlan.child(), mappedChild); + return MapperUtils.mapUnary(unary, mappedChild); } // From 8585f4d650acf4b29a2745278313f232a6a5f8e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20FOUCRET?= Date: Tue, 25 Nov 2025 15:07:40 +0100 Subject: [PATCH 02/12] Update docs/changelog/138586.yaml --- docs/changelog/138586.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/138586.yaml diff --git a/docs/changelog/138586.yaml b/docs/changelog/138586.yaml new file mode 100644 index 0000000000000..67fd7a9f40279 --- /dev/null +++ b/docs/changelog/138586.yaml @@ -0,0 +1,6 @@ +pr: 138586 +summary: Fix stats after `InferencePlan` (RERANK and COMPLETION) +area: ES|QL +type: bug +issues: + - 138582 From 41f2bd334c99151e1be9f091dfdd1106e41f8435 Mon Sep 17 00:00:00 2001 From: afoucret Date: Mon, 24 Nov 2025 11:51:06 +0100 Subject: [PATCH 03/12] Add INFERENCE_CCQ_SUPPORT capability --- .../elasticsearch/xpack/esql/action/EsqlCapabilities.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index ba3a8085b2820..43cee0c62260e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1688,6 +1688,11 @@ public enum Cap { * KNN function adds support for k and visit_percentage options */ KNN_FUNCTION_OPTIONS_K_VISIT_PERCENTAGE, + + /** + * Support for inference in cross-cluster queries (CCQ) + */ + INFERENCE_CCQ_SUPPORT, // Last capability should still have a comma for fewer merge conflicts when adding new ones :) // This comment prevents the semicolon from being on the previous capability when Spotless formats the file. ; From 09740c31dcefba7256895968ffb564f0827ace75 Mon Sep 17 00:00:00 2001 From: afoucret Date: Mon, 24 Nov 2025 11:59:35 +0100 Subject: [PATCH 04/12] Adding a new dense_vector_text CSV datasets and remove semantic_text dependency from text-embedding CSV tests. --- .../xpack/esql/CsvTestsDataLoader.java | 2 + .../main/resources/data/dense_vector_text.csv | 4 ++ .../resources/mapping-dense_vector_text.json | 12 ++++++ .../main/resources/text-embedding.csv-spec | 42 +++++++++---------- 4 files changed, 37 insertions(+), 23 deletions(-) create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/dense_vector_text.csv create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-dense_vector_text.json diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index bd42ee08ed384..fbe98e3204970 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -170,6 +170,7 @@ public class CsvTestsDataLoader { private static final TestDataset LOGS = new TestDataset("logs"); private static final TestDataset MV_TEXT = new TestDataset("mv_text"); private static final TestDataset DENSE_VECTOR = new TestDataset("dense_vector"); + private static final TestDataset DENSE_VECTOR_TEXT = new TestDataset("dense_vector_text"); private static final TestDataset COLORS = new TestDataset("colors"); private static final TestDataset COLORS_CMYK_LOOKUP = new TestDataset("colors_cmyk").withSetting("lookup-settings.json"); private static final TestDataset EXP_HISTO_SAMPLE = new TestDataset("exp_histo_sample"); @@ -241,6 +242,7 @@ public class CsvTestsDataLoader { Map.entry(COLORS_CMYK_LOOKUP.indexName, COLORS_CMYK_LOOKUP), Map.entry(MULTI_COLUMN_JOINABLE.indexName, MULTI_COLUMN_JOINABLE), Map.entry(MULTI_COLUMN_JOINABLE_LOOKUP.indexName, MULTI_COLUMN_JOINABLE_LOOKUP), + Map.entry(DENSE_VECTOR_TEXT.indexName, DENSE_VECTOR_TEXT), Map.entry(EXP_HISTO_SAMPLE.indexName, EXP_HISTO_SAMPLE) ); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/dense_vector_text.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/dense_vector_text.csv new file mode 100644 index 0000000000000..d189e5e5e376d --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/dense_vector_text.csv @@ -0,0 +1,4 @@ +_id:keyword,text_field:semantic_text,text_embedding_field:dense_vector +1,live long and prosper,[50.0, 57.0, 56.0] +2,all we have to decide is what to do with the time that is given to us, [45.0, 49.0, 57.0] +3,be excellent to each other,[45.0, 55.0, 54.0] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-dense_vector_text.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-dense_vector_text.json new file mode 100644 index 0000000000000..0b9fc264679b6 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-dense_vector_text.json @@ -0,0 +1,12 @@ +{ + "properties": { + "text_field": { + "type": "text" + }, + "text_embedding_field": { + "type": "dense_vector", + "dims": 3, + "similarity": "l2_norm" + } + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/text-embedding.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/text-embedding.csv-spec index 8011c665ce2e2..55006ee2f3cda 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/text-embedding.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/text-embedding.csv-spec @@ -26,44 +26,42 @@ Who is Victor Hugo? | [56.0, 50.0, 48.0] ; -text_embedding with knn on semantic_text_dense_field +text_embedding with knn on dense vector field required_capability: text_embedding_function required_capability: dense_vector_field_type_released required_capability: knn_function_v5 -required_capability: semantic_text_field_caps // tag::text-embedding-knn[] -FROM semantic_text METADATA _score +FROM dense_vector_text METADATA _score | EVAL query_embedding = TEXT_EMBEDDING("be excellent to each other", "test_dense_inference") -| WHERE KNN(semantic_text_dense_field, query_embedding) +| WHERE KNN(text_embedding_field, query_embedding) // end::text-embedding-knn[] | SORT _score DESC | LIMIT 10 -| KEEP semantic_text_field, query_embedding +| KEEP text_field, query_embedding ; -semantic_text_field:text | query_embedding:dense_vector +text_field:text | query_embedding:dense_vector be excellent to each other | [45.0, 55.0, 54.0] live long and prosper | [45.0, 55.0, 54.0] all we have to decide is what to do with the time that is given to us | [45.0, 55.0, 54.0] ; -text_embedding with knn (inline) on semantic_text_dense_field +text_embedding with knn (inline) on dense vector field required_capability: text_embedding_function required_capability: dense_vector_field_type_released required_capability: knn_function_v5 -required_capability: semantic_text_field_caps // tag::text-embedding-knn-inline[] -FROM semantic_text METADATA _score -| WHERE KNN(semantic_text_dense_field, TEXT_EMBEDDING("be excellent to each other", "test_dense_inference")) +FROM dense_vector_text METADATA _score +| WHERE KNN(text_embedding_field, TEXT_EMBEDDING("be excellent to each other", "test_dense_inference")) // end::text-embedding-knn-inline[] | SORT _score DESC | LIMIT 10 -| KEEP semantic_text_field +| KEEP text_field ; -semantic_text_field:text +text_field:text be excellent to each other live long and prosper all we have to decide is what to do with the time that is given to us @@ -74,16 +72,15 @@ required_capability: text_embedding_function required_capability: dense_vector_field_type_released required_capability: knn_function_v5 required_capability: fork_v9 -required_capability: semantic_text_field_caps -FROM semantic_text METADATA _score, _id -| WHERE KNN(semantic_text_dense_field, TEXT_EMBEDDING("be excellent to each other", "test_dense_inference")) OR KNN(semantic_text_dense_field, TEXT_EMBEDDING("live long and prosper", "test_dense_inference")) +FROM dense_vector_text METADATA _score, _id +| WHERE KNN(text_embedding_field, TEXT_EMBEDDING("be excellent to each other", "test_dense_inference")) OR KNN(text_embedding_field, TEXT_EMBEDDING("live long and prosper", "test_dense_inference")) | SORT _score DESC, _id | LIMIT 10 -| KEEP semantic_text_field +| KEEP text_field ; -semantic_text_field:text +text_field:text live long and prosper be excellent to each other all we have to decide is what to do with the time that is given to us @@ -94,17 +91,16 @@ required_capability: text_embedding_function required_capability: dense_vector_field_type_released required_capability: knn_function_v5 required_capability: fork_v9 -required_capability: semantic_text_field_caps -FROM semantic_text METADATA _score -| FORK (EVAL query_embedding = TEXT_EMBEDDING("be excellent to each other", "test_dense_inference") | WHERE KNN(semantic_text_dense_field, query_embedding)) - (EVAL query_embedding = TEXT_EMBEDDING("live long and prosper", "test_dense_inference") | WHERE KNN(semantic_text_dense_field, query_embedding)) +FROM dense_vector_text METADATA _score +| FORK (EVAL query_embedding = TEXT_EMBEDDING("be excellent to each other", "test_dense_inference") | WHERE KNN(text_embedding_field, query_embedding)) + (EVAL query_embedding = TEXT_EMBEDDING("live long and prosper", "test_dense_inference") | WHERE KNN(text_embedding_field, query_embedding)) | SORT _score DESC, _fork ASC | LIMIT 10 -| KEEP semantic_text_field, query_embedding, _fork +| KEEP text_field, query_embedding, _fork ; -semantic_text_field:text | query_embedding:dense_vector | _fork:keyword +text_field:text | query_embedding:dense_vector | _fork:keyword be excellent to each other | [45.0, 55.0, 54.0] | fork1 live long and prosper | [50.0, 57.0, 56.0] | fork2 live long and prosper | [45.0, 55.0, 54.0] | fork1 From b57333e878d8e8b03be365ae0ccb2dcc7db01dcb Mon Sep 17 00:00:00 2001 From: afoucret Date: Mon, 24 Nov 2025 18:19:01 +0100 Subject: [PATCH 05/12] Remove useless capability. --- .../qa/testFixtures/src/main/resources/text-embedding.csv-spec | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/text-embedding.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/text-embedding.csv-spec index 55006ee2f3cda..dfcb6f5e18bd8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/text-embedding.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/text-embedding.csv-spec @@ -71,7 +71,6 @@ text_embedding with multiple knn queries required_capability: text_embedding_function required_capability: dense_vector_field_type_released required_capability: knn_function_v5 -required_capability: fork_v9 FROM dense_vector_text METADATA _score, _id | WHERE KNN(text_embedding_field, TEXT_EMBEDDING("be excellent to each other", "test_dense_inference")) OR KNN(text_embedding_field, TEXT_EMBEDDING("live long and prosper", "test_dense_inference")) From 1c8ce5e92b2b3bcac4c0ec024db0408c73911b07 Mon Sep 17 00:00:00 2001 From: afoucret Date: Mon, 24 Nov 2025 18:20:16 +0100 Subject: [PATCH 06/12] Enable CCQ for inference commands. --- .../xpack/esql/parser/LogicalPlanBuilder.java | 8 ++++++-- .../xpack/esql/planner/mapper/Mapper.java | 3 ++- .../xpack/esql/parser/StatementParserTests.java | 12 ------------ 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index 1552108310480..7b474d5f3c770 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -1120,7 +1120,9 @@ public PlanFactory visitRerankCommand(EsqlBaseParser.RerankCommandContext ctx) { } return p -> { - checkForRemoteClusters(p, source, "RERANK"); + if (EsqlCapabilities.Cap.INFERENCE_CCQ_SUPPORT.isEnabled() == false) { + checkForRemoteClusters(p, source, "RERANK"); + } return applyRerankOptions(new Rerank(source, p, queryText, rerankFields, scoreAttribute), ctx.commandNamedParameters()); }; } @@ -1161,7 +1163,9 @@ public PlanFactory visitCompletionCommand(EsqlBaseParser.CompletionCommandContex } return p -> { - checkForRemoteClusters(p, source, "COMPLETION"); + if (EsqlCapabilities.Cap.INFERENCE_CCQ_SUPPORT.isEnabled() == false) { + checkForRemoteClusters(p, source, "COMPLETION"); + } return applyCompletionOptions(new Completion(source, p, prompt, targetField), ctx.commandNamedParameters()); }; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java index aff4f7d0c69a9..7bb4efb6bd51c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java @@ -24,7 +24,7 @@ import org.elasticsearch.xpack.esql.plan.logical.PipelineBreaker; import org.elasticsearch.xpack.esql.plan.logical.TopN; import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; -import org.elasticsearch.xpack.esql.plan.logical.inference.InferencePlan; +import org.elasticsearch.xpack.esql.plan.logical.inference.Rerank; import org.elasticsearch.xpack.esql.plan.logical.join.Join; import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig; import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; @@ -37,6 +37,7 @@ import org.elasticsearch.xpack.esql.plan.physical.MergeExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.plan.physical.TopNExec; +import org.elasticsearch.xpack.esql.plan.physical.inference.RerankExec; import org.elasticsearch.xpack.esql.session.Versioned; import java.util.List; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 421d032732945..6b73d4a62dcaf 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -4220,12 +4220,6 @@ public void testInvalidRerank() { expectError("FROM foo* | RERANK ON title WITH inferenceId", "line 1:20: extraneous input 'ON' expecting {QUOTED_STRING"); expectError("FROM foo* | RERANK \"query text\" WITH inferenceId", "line 1:33: mismatched input 'WITH' expecting 'on'"); - var fromPatterns = randomIndexPatterns(CROSS_CLUSTER); - expectError( - "FROM " + fromPatterns + " | RERANK \"query text\" ON title WITH { \"inference_id\" : \"inference_id\" }", - "invalid index pattern [" + unquoteIndexPattern(fromPatterns) + "], remote clusters are not supported with RERANK" - ); - expectError( "FROM foo* | RERANK \"query text\" ON title WITH { \"inference_id\": { \"a\": 123 } }", "Option [inference_id] must be a valid string, found [{ \"a\": 123 }]" @@ -4311,12 +4305,6 @@ public void testInvalidCompletion() { expectError("FROM foo* | COMPLETION completion=prompt WITH", "ine 1:46: mismatched input '' expecting '{'"); - var fromPatterns = randomIndexPatterns(CROSS_CLUSTER); - expectError( - "FROM " + fromPatterns + " | COMPLETION prompt_field WITH { \"inference_id\" : \"inference_id\" }", - "invalid index pattern [" + unquoteIndexPattern(fromPatterns) + "], remote clusters are not supported with COMPLETION" - ); - expectError( "FROM foo* | COMPLETION prompt WITH { \"inference_id\": { \"a\": 123 } }", "line 1:54: Option [inference_id] must be a valid string, found [{ \"a\": 123 }]" From a3d4e64d464b9d93353c44e93514c2ff6891571a Mon Sep 17 00:00:00 2001 From: afoucret Date: Mon, 24 Nov 2025 18:20:40 +0100 Subject: [PATCH 07/12] Enable CCQ tests for inference commands. --- .../qa/server/multi-clusters/build.gradle | 2 ++ .../xpack/esql/ccq/Clusters.java | 19 +++++++++++++ .../xpack/esql/ccq/MultiClusterSpecIT.java | 28 +++++++++++++++++-- .../single_node/PushExpressionToLoadIT.java | 2 +- .../esql/qa/single_node/PushQueriesIT.java | 2 +- .../xpack/esql/qa/rest/EsqlSpecTestCase.java | 28 ++++++++++++++++--- .../esql/qa/rest/SemanticMatchTestCase.java | 8 +++--- .../xpack/esql/CsvTestsDataLoader.java | 6 ++-- 8 files changed, 80 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle b/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle index a44c33f72ada4..bd46073035979 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle +++ b/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle @@ -21,6 +21,8 @@ dependencies { javaRestTestImplementation project(xpackModule('esql:qa:testFixtures')) javaRestTestImplementation project(xpackModule('esql:qa:server')) javaRestTestImplementation project(xpackModule('esql')) + + clusterPlugins project(':x-pack:plugin:inference:qa:test-service-plugin') } def supportedVersion = bwcVersion -> { diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/Clusters.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/Clusters.java index 76b52708b4fac..1747c8bffa43b 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/Clusters.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/Clusters.java @@ -54,6 +54,9 @@ public static ElasticsearchCluster localCluster(ElasticsearchCluster remoteClust if (supportRetryOnShardFailures(version) == false) { cluster.setting("cluster.routing.rebalance.enable", "none"); } + if (localClusterSupportsInferenceTestService()) { + cluster.plugin("inference-service-test"); + } return cluster.build(); } @@ -73,6 +76,22 @@ public static org.elasticsearch.Version bwcVersion() { return local.before(remote) ? local : remote; } + public static boolean localClusterSupportsInferenceTestService() { + return isNewToOld() && localClusterVersion().onOrAfter(org.elasticsearch.Version.fromString("9.3.0")); + } + + /** + * Returns true if the current task is a "newToOld" BWC test. + * Checks the tests.task system property to determine the task type. + */ + public static boolean isNewToOld() { + String taskName = System.getProperty("tests.task"); + if (taskName == null) { + return false; + } + return taskName.endsWith("#newToOld"); + } + private static Version distributionVersion(String key) { final String val = System.getProperty(key); return val != null ? Version.fromString(val) : Version.CURRENT; diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index 3a6b3a6698364..b87859ebaffc3 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -48,6 +48,7 @@ import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.CSV_DATASET_MAP; import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.ENRICH_SOURCE_INDICES; import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.COMPLETION; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.ENABLE_LOOKUP_JOIN_ON_REMOTE; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.FORK_V9; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINE_STATS; @@ -55,7 +56,9 @@ import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.RERANK; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.TEXT_EMBEDDING_FUNCTION; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.UNMAPPED_FIELDS; import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.hasCapabilities; import static org.mockito.ArgumentMatchers.any; @@ -81,6 +84,12 @@ public class MultiClusterSpecIT extends EsqlSpecTestCase { private static RestClient remoteClusterClient; private static DataLocation dataLocation = null; + private static final Set LOCAL_ONLY_INFERENCE_CAPABILITIES = Set.of( + RERANK.capabilityName(), + COMPLETION.capabilityName(), + TEXT_EMBEDDING_FUNCTION.capabilityName() + ); + @ParametersFactory(argumentFormatting = "csv-spec:%2$s.%3$s") public static List readScriptSpec() throws Exception { List urls = classpathResources("/*.csv-spec"); @@ -133,8 +142,16 @@ protected void shouldSkipTest(String testName) throws IOException { .filter(c -> c.equals("metadata_fields_remote_test") == false) .toList(); } + + // Check all capabilities on the local cluster first. super.shouldSkipTest(testName); - checkCapabilities(remoteClusterClient(), remoteFeaturesService(), testName, testCase); + + // Filter out capabilities that are required only on the local cluster and then check the remaining on the remote cluster. + List remoteCapabilities = testCase.requiredCapabilities.stream() + .filter(c -> LOCAL_ONLY_INFERENCE_CAPABILITIES.contains(c) == false) + .toList(); + checkCapabilities(remoteClusterClient(), remoteFeaturesService(), testName, remoteCapabilities); + // Do not run tests including "METADATA _index" unless marked with metadata_fields_remote_test, // because they may produce inconsistent results with multiple clusters. assumeFalse("can't test with _index metadata", (remoteMetadata == false) && hasIndexMetadata(testCase.query)); @@ -230,10 +247,11 @@ protected RestClient buildClient(Settings settings, HttpHost[] localHosts) throw .collect(Collectors.toSet()); /** - * Creates a new mock client that dispatches every request to both the local and remote clusters, excluding _bulk and _query requests. + * Creates a new mock client that dispatches every request to both the local and remote clusters, excluding _bulk, _query, and _inference requests. * - '_bulk' requests are randomly sent to either the local or remote cluster to populate data. Some spec tests, such as AVG, * prevent the splitting of bulk requests. * - '_query' requests are dispatched to the local cluster only, as we are testing cross-cluster queries. + * - '_inference' requests are dispatched to the local cluster only, as inference endpoints are not available on remote clusters. */ static RestClient twoClients(RestClient localClient, RestClient remoteClient) throws IOException { RestClient twoClients = mock(RestClient.class); @@ -245,6 +263,8 @@ static RestClient twoClients(RestClient localClient, RestClient remoteClient) th String endpoint = request.getEndpoint(); if (endpoint.startsWith("/_query")) { return localClient.performRequest(request); + } else if (endpoint.startsWith("/_inference")) { + return localClient.performRequest(request); } else if (endpoint.endsWith("/_bulk") && METADATA_INDICES.stream().anyMatch(i -> endpoint.equals("/" + i + "/_bulk"))) { return remoteClient.performRequest(request); } else if (endpoint.endsWith("/_bulk") @@ -409,6 +429,10 @@ protected boolean supportsInferenceTestService() { return false; } + protected boolean supportsInferenceTestServiceOnLocalCluster() { + return Clusters.localClusterSupportsInferenceTestService(); + } + @Override protected boolean supportsIndexModeLookup() throws IOException { return hasCapabilities(adminClient(), List.of(JOIN_LOOKUP_V12.capabilityName())); diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushExpressionToLoadIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushExpressionToLoadIT.java index 05e22cbcaa124..7bb794f72995b 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushExpressionToLoadIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushExpressionToLoadIT.java @@ -594,7 +594,7 @@ protected boolean preserveClusterUponCompletion() { private void setUpTextEmbeddingInferenceEndpoint() throws IOException { setupEmbeddings = true; - Request request = new Request("PUT", "_inference/text_embedding/test"); + Request request = new Request("PUT", "/_inference/text_embedding/test"); request.setJsonEntity(""" { "service": "text_embedding_test_service", diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java index 0076ee4d9fb43..48a58a282c896 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java @@ -547,7 +547,7 @@ public void setUpTextEmbeddingInferenceEndpoint() throws IOException { return; } setupEmbeddings = true; - Request request = new Request("PUT", "_inference/text_embedding/test"); + Request request = new Request("PUT", "/_inference/text_embedding/test"); request.setJsonEntity(""" { "service": "text_embedding_test_service", diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java index 38b7822479c65..eea03e500c571 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java @@ -175,7 +175,8 @@ public void setup() { assumeTrue("test clusters were broken", testClustersOk); INGEST.protectedBlock(() -> { // Inference endpoints must be created before ingesting any datasets that rely on them (mapping of inference_id) - if (supportsInferenceTestService()) { + // If multiple clusters are used, only create endpoints on the local cluster if it supports the inference test service. + if (supportsInferenceTestServiceOnLocalCluster()) { createInferenceEndpoints(adminClient()); } loadDataSetIntoEs( @@ -238,6 +239,9 @@ protected void shouldSkipTest(String testName) throws IOException { if (requiresInferenceEndpoint()) { assumeTrue("Inference test service needs to be supported", supportsInferenceTestService()); } + if (requiresInferenceEndpointOnLocalCluster()) { + assumeTrue("Inference test service needs to be supported", supportsInferenceTestServiceOnLocalCluster()); + } checkCapabilities(adminClient(), testFeatureService, testName, testCase); assumeTrue("Test " + testName + " is not enabled", isEnabled(testName, instructions, Version.CURRENT)); if (supportsSourceFieldMapping() == false) { @@ -251,13 +255,22 @@ protected static void checkCapabilities( String testName, CsvTestCase testCase ) { - if (hasCapabilities(client, testCase.requiredCapabilities)) { + checkCapabilities(client, testFeatureService, testName, testCase.requiredCapabilities); + } + + protected static void checkCapabilities( + RestClient client, + TestFeatureService testFeatureService, + String testName, + List requiredCapabilities + ) { + if (hasCapabilities(client, requiredCapabilities)) { return; } var features = new EsqlFeatures().getFeatures().stream().map(NodeFeature::id).collect(Collectors.toSet()); - for (String feature : testCase.requiredCapabilities) { + for (String feature : requiredCapabilities) { var esqlFeature = "esql." + feature; assumeTrue("Requested capability " + feature + " is an ESQL cluster feature", features.contains(esqlFeature)); assumeTrue("Test " + testName + " requires " + feature, testFeatureService.clusterHasFeature(esqlFeature)); @@ -268,9 +281,16 @@ protected boolean supportsInferenceTestService() { return true; } + protected boolean supportsInferenceTestServiceOnLocalCluster() { + return supportsInferenceTestService(); + } + protected boolean requiresInferenceEndpoint() { + return testCase.requiredCapabilities.contains(SEMANTIC_TEXT_FIELD_CAPS.capabilityName()); + } + + protected boolean requiresInferenceEndpointOnLocalCluster() { return Stream.of( - SEMANTIC_TEXT_FIELD_CAPS.capabilityName(), RERANK.capabilityName(), COMPLETION.capabilityName(), KNN_FUNCTION_V5.capabilityName(), diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/SemanticMatchTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/SemanticMatchTestCase.java index aada75f151d66..d2559116d3f20 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/SemanticMatchTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/SemanticMatchTestCase.java @@ -143,7 +143,7 @@ public void setUpIndices() throws IOException { @Before public void setUpSparseEmbeddingInferenceEndpoint() throws IOException { - Request request = new Request("PUT", "_inference/sparse_embedding/test_sparse_inference"); + Request request = new Request("PUT", "/_inference/sparse_embedding/test_sparse_inference"); request.setJsonEntity(""" { "service": "test_service", @@ -165,7 +165,7 @@ public void setUpSparseEmbeddingInferenceEndpoint() throws IOException { @Before public void setUpTextEmbeddingInferenceEndpoint() throws IOException { - Request request = new Request("PUT", "_inference/text_embedding/test_dense_inference"); + Request request = new Request("PUT", "/_inference/text_embedding/test_dense_inference"); request.setJsonEntity(""" { "service": "text_embedding_test_service", @@ -191,7 +191,7 @@ public void wipeData() throws IOException { adminClient().performRequest(new Request("DELETE", "*")); try { - adminClient().performRequest(new Request("DELETE", "_inference/test_sparse_inference")); + adminClient().performRequest(new Request("DELETE", "/_inference/test_sparse_inference")); } catch (ResponseException e) { // 404 here means the endpoint was not created if (e.getResponse().getStatusLine().getStatusCode() != 404) { @@ -200,7 +200,7 @@ public void wipeData() throws IOException { } try { - adminClient().performRequest(new Request("DELETE", "_inference/test_dense_inference")); + adminClient().performRequest(new Request("DELETE", "/_inference/test_dense_inference")); } catch (ResponseException e) { // 404 here means the endpoint was not created if (e.getResponse().getStatusLine().getStatusCode() != 404) { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index fbe98e3204970..2c1fcc77493f4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -585,13 +585,13 @@ public static boolean clusterHasCompletionInferenceEndpoint(RestClient client) t private static void createInferenceEndpoint(RestClient client, TaskType taskType, String inferenceId, String modelSettings) throws IOException { - Request request = new Request("PUT", "_inference/" + taskType.name() + "/" + inferenceId); + Request request = new Request("PUT", "/_inference/" + taskType.name() + "/" + inferenceId); request.setJsonEntity(modelSettings); client.performRequest(request); } private static boolean clusterHasInferenceEndpoint(RestClient client, TaskType taskType, String inferenceId) throws IOException { - Request request = new Request("GET", "_inference/" + taskType.name() + "/" + inferenceId); + Request request = new Request("GET", "/_inference/" + taskType.name() + "/" + inferenceId); try { client.performRequest(request); } catch (ResponseException e) { @@ -605,7 +605,7 @@ private static boolean clusterHasInferenceEndpoint(RestClient client, TaskType t private static void deleteInferenceEndpoint(RestClient client, String inferenceId) throws IOException { try { - client.performRequest(new Request("DELETE", "_inference/" + inferenceId)); + client.performRequest(new Request("DELETE", "/_inference/" + inferenceId)); } catch (ResponseException e) { // 404 here means the endpoint was not created if (e.getResponse().getStatusLine().getStatusCode() != 404) { From 360714bead500a6d58457f48a9db314c5750a72d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20FOUCRET?= Date: Mon, 24 Nov 2025 18:27:07 +0100 Subject: [PATCH 08/12] Update docs/changelog/138522.yaml --- docs/changelog/138522.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/138522.yaml diff --git a/docs/changelog/138522.yaml b/docs/changelog/138522.yaml new file mode 100644 index 0000000000000..7fa503ca452e3 --- /dev/null +++ b/docs/changelog/138522.yaml @@ -0,0 +1,6 @@ +pr: 138522 +summary: Support CCQ for inference commands +area: ES|QL +type: enhancement +issues: + - 136860 From f11810ef0cb84f431133a549d069a2d2d4e6cacf Mon Sep 17 00:00:00 2001 From: afoucret Date: Tue, 25 Nov 2025 08:06:12 +0100 Subject: [PATCH 09/12] Checkstyle. --- .../org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index b87859ebaffc3..df12b7e814907 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -247,7 +247,8 @@ protected RestClient buildClient(Settings settings, HttpHost[] localHosts) throw .collect(Collectors.toSet()); /** - * Creates a new mock client that dispatches every request to both the local and remote clusters, excluding _bulk, _query, and _inference requests. + * Creates a new mock client that dispatches every request to both the local and remote clusters, excluding _bulk, _query, + * and _inference requests : * - '_bulk' requests are randomly sent to either the local or remote cluster to populate data. Some spec tests, such as AVG, * prevent the splitting of bulk requests. * - '_query' requests are dispatched to the local cluster only, as we are testing cross-cluster queries. From 01a6622bb12d1f511492c18ca6e91ea9fd108d8b Mon Sep 17 00:00:00 2001 From: afoucret Date: Tue, 25 Nov 2025 09:14:40 +0100 Subject: [PATCH 10/12] Updating doc to match examples. --- .../esql/_snippets/functions/examples/text_embedding.md | 8 ++++---- .../esql/kibana/definition/functions/text_embedding.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/text_embedding.md b/docs/reference/query-languages/esql/_snippets/functions/examples/text_embedding.md index a6384e1222891..fa6911deef291 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/text_embedding.md +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/text_embedding.md @@ -12,16 +12,16 @@ ROW input="Who is Victor Hugo?" Generate text embeddings and store them in a variable for reuse in KNN vector search queries. ```esql -FROM semantic_text METADATA _score +FROM dense_vector_text METADATA _score | EVAL query_embedding = TEXT_EMBEDDING("be excellent to each other", "test_dense_inference") -| WHERE KNN(semantic_text_dense_field, query_embedding) +| WHERE KNN(text_embedding_field, query_embedding) ``` Directly embed text within a KNN query for streamlined vector search without intermediate variables. ```esql -FROM semantic_text METADATA _score -| WHERE KNN(semantic_text_dense_field, TEXT_EMBEDDING("be excellent to each other", "test_dense_inference")) +FROM dense_vector_text METADATA _score +| WHERE KNN(text_embedding_field, TEXT_EMBEDDING("be excellent to each other", "test_dense_inference")) ``` diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/text_embedding.json b/docs/reference/query-languages/esql/kibana/definition/functions/text_embedding.json index f2c186a90cda7..6712986e4232a 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/text_embedding.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/text_embedding.json @@ -25,8 +25,8 @@ ], "examples" : [ "ROW input=\"Who is Victor Hugo?\"\n| EVAL embedding = TEXT_EMBEDDING(\"Who is Victor Hugo?\", \"test_dense_inference\")", - "FROM semantic_text METADATA _score\n| EVAL query_embedding = TEXT_EMBEDDING(\"be excellent to each other\", \"test_dense_inference\")\n| WHERE KNN(semantic_text_dense_field, query_embedding)", - "FROM semantic_text METADATA _score\n| WHERE KNN(semantic_text_dense_field, TEXT_EMBEDDING(\"be excellent to each other\", \"test_dense_inference\"))" + "FROM dense_vector_text METADATA _score\n| EVAL query_embedding = TEXT_EMBEDDING(\"be excellent to each other\", \"test_dense_inference\")\n| WHERE KNN(text_embedding_field, query_embedding)", + "FROM dense_vector_text METADATA _score\n| WHERE KNN(text_embedding_field, TEXT_EMBEDDING(\"be excellent to each other\", \"test_dense_inference\"))" ], "preview" : true, "snapshot_only" : false From 46d8c4b08076dc3041acb5170f9c7aba806eb6b7 Mon Sep 17 00:00:00 2001 From: afoucret Date: Tue, 25 Nov 2025 10:04:44 +0100 Subject: [PATCH 11/12] Adding missing cap to KNN csv tests (semantic_text) --- .../qa/testFixtures/src/main/resources/knn-function.csv-spec | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec index 905952e0a2ce6..19c9bb4663437 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec @@ -338,6 +338,7 @@ golden rod testKnnWithSemanticText required_capability: knn_function_v5 +required_capability: semantic_text_field_caps from semantic_text | where knn(semantic_text_dense_field, [0, 1, 2]) @@ -353,6 +354,7 @@ live long and prosper testKnnWithSemanticTextAndKeyword required_capability: knn_function_v5 +required_capability: semantic_text_field_caps from semantic_text | where knn(semantic_text_dense_field, [0, 1, 2]) @@ -369,6 +371,7 @@ be excellent to each other | host3 testKnnWithSemanticTextMultiValueField required_capability: knn_function_v5 +required_capability: semantic_text_field_caps from semantic_text metadata _id | where match(st_multi_value, "something") AND match(host, "host1") @@ -381,6 +384,7 @@ _id: keyword | st_multi_value:text testKnnWithSemanticTextWithEvalsAndOtherFunctionsAndStats required_capability: knn_function_v5 +required_capability: semantic_text_field_caps from semantic_text | where qstr("description:some*") @@ -397,6 +401,7 @@ result:long testKnnWithSemanticTextAndKql required_capability: knn_function_v5 required_capability: kql_function +required_capability: semantic_text_field_caps from semantic_text | where kql("host:host1") AND knn(semantic_text_dense_field, [0, 1, 2]) From b119cf65c830d98663cf1b812ffc43b8b33ff26d Mon Sep 17 00:00:00 2001 From: afoucret Date: Wed, 26 Nov 2025 08:49:25 +0100 Subject: [PATCH 12/12] Fix error introduced while rebasing. --- .../org/elasticsearch/xpack/esql/planner/mapper/Mapper.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java index 7bb4efb6bd51c..aff4f7d0c69a9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java @@ -24,7 +24,7 @@ import org.elasticsearch.xpack.esql.plan.logical.PipelineBreaker; import org.elasticsearch.xpack.esql.plan.logical.TopN; import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; -import org.elasticsearch.xpack.esql.plan.logical.inference.Rerank; +import org.elasticsearch.xpack.esql.plan.logical.inference.InferencePlan; import org.elasticsearch.xpack.esql.plan.logical.join.Join; import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig; import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; @@ -37,7 +37,6 @@ import org.elasticsearch.xpack.esql.plan.physical.MergeExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.plan.physical.TopNExec; -import org.elasticsearch.xpack.esql.plan.physical.inference.RerankExec; import org.elasticsearch.xpack.esql.session.Versioned; import java.util.List;