diff --git a/docs/changelog/137713.yaml b/docs/changelog/137713.yaml new file mode 100644 index 0000000000000..7dffe59f45f58 --- /dev/null +++ b/docs/changelog/137713.yaml @@ -0,0 +1,6 @@ +pr: 137713 +summary: TS Disallow renaming into timestamp prior to implicit use +area: ES|QL +type: bug +issues: + - 137655 diff --git a/muted-tests.yml b/muted-tests.yml index d16122b4cc472..fa37b26fd14cd 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -417,9 +417,6 @@ tests: - class: org.elasticsearch.xpack.ilm.CCRIndexLifecycleIT method: testTsdbLeaderIndexRolloverAndSyncAfterWaitUntilEndTime {targetCluster=FOLLOWER} issue: https://github.com/elastic/elasticsearch/issues/137565 -- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeMetricsIT - method: test - issue: https://github.com/elastic/elasticsearch/issues/137655 - class: org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilterBasicLicenseIT method: testLicenseInvalidForInference {p0=true} issue: https://github.com/elastic/elasticsearch/issues/137690 diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index c5db80ac1ae51..26468bff22877 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -82,6 +82,7 @@ public abstract class GenerativeRestTest extends ESRestTestCase implements Query "implicit time-series aggregation function .* doesn't support type .*", "INLINE STATS .* can only be used after STATS when used with TS command", "cannot group by a metric field .* in a time-series aggregation", + "a @timestamp field of type date or date_nanos to be present when run with the TS command, but it was not present", "Output has changed from \\[.*\\] to \\[.*\\]" // https://github.com/elastic/elasticsearch/issues/134794 ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Delta.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Delta.java index 19636e5845a04..c9e28bf67e0b3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Delta.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Delta.java @@ -123,7 +123,7 @@ public Delta perTimeSeriesAggregation() { @Override public String toString() { - return "delta(" + field() + ")"; + return "delta(" + field() + ", " + timestamp() + ")"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTime.java index 67ab4de0dc17a..70b0a2d7a3c11 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTime.java @@ -141,7 +141,7 @@ public FirstOverTime perTimeSeriesAggregation() { @Override public String toString() { - return "first_over_time(" + field() + ")"; + return "first_over_time(" + field() + ", " + timestamp() + ")"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Idelta.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Idelta.java index c70a1b4487b76..85194ae0a9d04 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Idelta.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Idelta.java @@ -125,7 +125,7 @@ public Idelta perTimeSeriesAggregation() { @Override public String toString() { - return "idelta(" + field() + ")"; + return "idelta(" + field() + ", " + timestamp() + ")"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Increase.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Increase.java index 543ecf97327d7..8fcc29b98a443 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Increase.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Increase.java @@ -126,7 +126,7 @@ public Increase perTimeSeriesAggregation() { @Override public String toString() { - return "increase(" + field() + ")"; + return "increase(" + field() + ", " + timestamp() + ")"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Irate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Irate.java index 74bc8082beb9d..e4f003d4b9310 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Irate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Irate.java @@ -122,7 +122,7 @@ public Irate perTimeSeriesAggregation() { @Override public String toString() { - return "irate(" + field() + ")"; + return "irate(" + field() + ", " + timestamp() + ")"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java index 4469a44242ecd..202b9d236fe58 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java @@ -143,7 +143,7 @@ public LastOverTime perTimeSeriesAggregation() { @Override public String toString() { - return "last_over_time(" + field() + ")"; + return "last_over_time(" + field() + ", " + timestamp() + ")"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java index 48880199f188b..4edf74bc30a61 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java @@ -126,7 +126,7 @@ public Rate perTimeSeriesAggregation() { @Override public String toString() { - return "rate(" + field() + ")"; + return "rate(" + field() + ", " + timestamp() + ")"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TranslateTimeSeriesAggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TranslateTimeSeriesAggregate.java index dbdaa2aab2df7..289bb4fe0eda0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TranslateTimeSeriesAggregate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TranslateTimeSeriesAggregate.java @@ -16,9 +16,11 @@ import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.core.util.Holder; +import org.elasticsearch.xpack.esql.expression.function.TimestampAware; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.DimensionValues; import org.elasticsearch.xpack.esql.expression.function.aggregate.LastOverTime; @@ -240,6 +242,28 @@ protected LogicalPlan rule(TimeSeriesAggregate aggregate, LogicalOptimizerContex } } } + if (aggregate.child().output().contains(timestamp.get()) == false) { + var timestampAwareFunctions = timeSeriesAggs.keySet() + .stream() + .filter(ts -> ts instanceof TimestampAware) + .map(Node::sourceText) + .sorted() + .toList(); + if (timestampAwareFunctions.isEmpty() == false) { + int size = timestampAwareFunctions.size(); + throw new IllegalArgumentException( + "Function" + + (size > 1 ? "s " : " ") + + "[" + + String.join(", ", timestampAwareFunctions.subList(0, Math.min(size, 3))) + + (size > 3 ? ", ..." : "") + + "] require" + + (size > 1 ? " " : "s ") + + "a @timestamp field of type date or date_nanos to be present when run with the TS command, " + + "but it was not present." + ); + } + } // time-series aggregates must be grouped by _tsid (and time-bucket) first and re-group by users key List firstPassGroupings = new ArrayList<>(); firstPassGroupings.add(tsid.get()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 736075f9f5015..efba64153aebe 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -9670,4 +9670,52 @@ public void testFullTextFunctionOnEvalNull() { EsRelation relation = as(filter.child(), EsRelation.class); assertEquals("test", relation.indexPattern()); } + + /* + * Renaming or shadowing the @timestamp field prior to running stats with TS command is not allowed. + */ + public void testTranslateMetricsAfterRenamingTimestamp() { + assertThat( + expectThrows( + IllegalArgumentException.class, + () -> logicalOptimizerWithLatestVersion.optimize(metricsAnalyzer.analyze(parser.createStatement(""" + TS k8s | + EVAL @timestamp = region | + STATS max(network.cost), count(network.eth0.rx) + """))) + ).getMessage(), + containsString(""" + Functions [count(network.eth0.rx), max(network.cost)] require a @timestamp field of type date or date_nanos \ + to be present when run with the TS command, but it was not present.""") + ); + + assertThat( + expectThrows( + IllegalArgumentException.class, + () -> logicalOptimizerWithLatestVersion.optimize(metricsAnalyzer.analyze(parser.createStatement(""" + TS k8s | + DISSECT event "%{@timestamp} %{network.total_bytes_in}" | + STATS ohxqxpSqEZ = avg(network.eth0.currently_connected_clients) + """))) + ).getMessage(), + containsString(""" + Function [avg(network.eth0.currently_connected_clients)] requires a @timestamp field of type date or date_nanos \ + to be present when run with the TS command, but it was not present.""") + ); + + // we may want to allow this later + assertThat( + expectThrows( + IllegalArgumentException.class, + () -> logicalOptimizerWithLatestVersion.optimize(metricsAnalyzer.analyze(parser.createStatement(""" + TS k8s | + EVAL `@timestamp` = @timestamp + 1day | + STATS std_dev(network.eth0.currently_connected_clients) + """))) + ).getMessage(), + containsString(""" + Function [std_dev(network.eth0.currently_connected_clients)] requires a @timestamp field of type date or date_nanos \ + to be present when run with the TS command, but it was not present.""") + ); + } }