Skip to content

Commit 4664e65

Browse files
davidkyledavidkyle
authored
Merge branch 'main' into unmute-ccs-tests
2 parents c4812ad + 7f01282 commit 4664e65

File tree

267 files changed

+5418
-3990
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

267 files changed

+5418
-3990
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@ public class QueryPlanningBenchmark {
7171
}
7272

7373
private PlanTelemetry telemetry;
74-
private EsqlParser defaultParser;
7574
private Analyzer manyFieldsAnalyzer;
7675
private LogicalPlanOptimizer defaultOptimizer;
7776
private Configuration config;
@@ -111,7 +110,6 @@ public void setup() {
111110
TransportVersion minimumVersion = TransportVersion.current();
112111

113112
telemetry = new PlanTelemetry(functionRegistry);
114-
defaultParser = new EsqlParser();
115113
manyFieldsAnalyzer = new Analyzer(
116114
new AnalyzerContext(
117115
config,
@@ -128,14 +126,14 @@ public void setup() {
128126
}
129127

130128
private LogicalPlan plan(EsqlParser parser, Analyzer analyzer, LogicalPlanOptimizer optimizer, String query) {
131-
var parsed = parser.createStatement(query, new QueryParams(), telemetry);
129+
var parsed = parser.parseQuery(query, new QueryParams(), telemetry);
132130
var analyzed = analyzer.analyze(parsed);
133131
var optimized = optimizer.optimize(analyzed);
134132
return optimized;
135133
}
136134

137135
@Benchmark
138136
public void manyFields(Blackhole blackhole) {
139-
blackhole.consume(plan(defaultParser, manyFieldsAnalyzer, defaultOptimizer, "FROM test | LIMIT 10"));
137+
blackhole.consume(plan(EsqlParser.INSTANCE, manyFieldsAnalyzer, defaultOptimizer, "FROM test | LIMIT 10"));
140138
}
141139
}

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmArgumentParsingSystemMemoryInfo.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.server.cli;
11+
12+
import java.util.List;
13+
14+
public abstract class JvmArgumentParsingSystemMemoryInfo implements SystemMemoryInfo {
15+
private final List<String> userDefinedJvmOptions;
16+
17+
public JvmArgumentParsingSystemMemoryInfo(List<String> userDefinedJvmOptions) {
18+
this.userDefinedJvmOptions = userDefinedJvmOptions;
19+
}
20+
21+
protected long getBytesFromSystemProperty(String systemProperty, long defaultValue) {
22+
return userDefinedJvmOptions.stream()
23+
.filter(option -> option.startsWith("-D" + systemProperty + "="))
24+
.map(totalMemoryOverheadBytesOption -> {
25+
try {
26+
long bytes = Long.parseLong(totalMemoryOverheadBytesOption.split("=", 2)[1]);
27+
if (bytes < 0) {
28+
throw new IllegalArgumentException("Negative bytes size specified in [" + totalMemoryOverheadBytesOption + "]");
29+
}
30+
return bytes;
31+
} catch (NumberFormatException e) {
32+
throw new IllegalArgumentException("Unable to parse number of bytes from [" + totalMemoryOverheadBytesOption + "]", e);
33+
}
34+
})
35+
.reduce((previous, current) -> current) // this is effectively findLast(), so that ES_JAVA_OPTS overrides jvm.options
36+
.orElse(defaultValue);
37+
}
38+
}

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,10 @@ private List<String> jvmOptions(
142142
}
143143

144144
final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
145-
final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(substitutedJvmOptions, new DefaultSystemMemoryInfo());
145+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(
146+
substitutedJvmOptions,
147+
new OverridableSystemMemoryInfo(substitutedJvmOptions, new DefaultSystemMemoryInfo())
148+
);
146149
substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(args.nodeSettings(), memoryInfo, substitutedJvmOptions));
147150
final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions, args.nodeSettings());
148151
final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions(args.nodeSettings(), cliSysprops);

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/OverheadSystemMemoryInfo.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.server.cli;
11+
12+
import java.util.List;
13+
14+
/**
15+
* A {@link SystemMemoryInfo} implementation that reduces the reported available system memory by a set amount. This is intended to account
16+
* for overhead cost of other bundled Elasticsearch processes, such as the CLI tool launcher. By default, this is
17+
* {@link org.elasticsearch.server.cli.OverheadSystemMemoryInfo#SERVER_CLI_OVERHEAD } but can be overridden via the
18+
* {@code es.total_memory_overhead_bytes} system property.
19+
*/
20+
public class OverheadSystemMemoryInfo extends JvmArgumentParsingSystemMemoryInfo {
21+
static final long SERVER_CLI_OVERHEAD = 100 * 1024L * 1024L;
22+
23+
private final SystemMemoryInfo delegate;
24+
25+
public OverheadSystemMemoryInfo(List<String> userDefinedJvmOptions, SystemMemoryInfo delegate) {
26+
super(userDefinedJvmOptions);
27+
this.delegate = delegate;
28+
}
29+
30+
@Override
31+
public long availableSystemMemory() {
32+
long overheadBytes = getBytesFromSystemProperty("es.total_memory_overhead_bytes", SERVER_CLI_OVERHEAD);
33+
return delegate.availableSystemMemory() - overheadBytes;
34+
}
35+
}

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/OverridableSystemMemoryInfo.java‎

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -17,33 +17,17 @@
1717
* has been specified using the {@code es.total_memory_bytes} system property, or
1818
* else returns the value provided by a fallback provider.
1919
*/
20-
public final class OverridableSystemMemoryInfo implements SystemMemoryInfo {
20+
public final class OverridableSystemMemoryInfo extends JvmArgumentParsingSystemMemoryInfo {
2121

22-
private final List<String> userDefinedJvmOptions;
2322
private final SystemMemoryInfo fallbackSystemMemoryInfo;
2423

2524
public OverridableSystemMemoryInfo(final List<String> userDefinedJvmOptions, SystemMemoryInfo fallbackSystemMemoryInfo) {
26-
this.userDefinedJvmOptions = Objects.requireNonNull(userDefinedJvmOptions);
25+
super(userDefinedJvmOptions);
2726
this.fallbackSystemMemoryInfo = Objects.requireNonNull(fallbackSystemMemoryInfo);
2827
}
2928

3029
@Override
3130
public long availableSystemMemory() {
32-
33-
return userDefinedJvmOptions.stream()
34-
.filter(option -> option.startsWith("-Des.total_memory_bytes="))
35-
.map(totalMemoryBytesOption -> {
36-
try {
37-
long bytes = Long.parseLong(totalMemoryBytesOption.split("=", 2)[1]);
38-
if (bytes < 0) {
39-
throw new IllegalArgumentException("Negative memory size specified in [" + totalMemoryBytesOption + "]");
40-
}
41-
return bytes;
42-
} catch (NumberFormatException e) {
43-
throw new IllegalArgumentException("Unable to parse number of bytes from [" + totalMemoryBytesOption + "]", e);
44-
}
45-
})
46-
.reduce((previous, current) -> current) // this is effectively findLast(), so that ES_JAVA_OPTS overrides jvm.options
47-
.orElse(fallbackSystemMemoryInfo.availableSystemMemory());
31+
return getBytesFromSystemProperty("es.total_memory_bytes", fallbackSystemMemoryInfo.availableSystemMemory());
4832
}
4933
}

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,8 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
118118
return;
119119
}
120120

121+
// Call the GC to try and free up as much heap as we can since we don't intend to do much if any more allocation after this
122+
System.gc();
121123
// we are running in the foreground, so wait for the server to exit
122124
int exitCode = server.waitFor();
123125
onExit(exitCode);

‎distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/OverheadSystemMemoryInfoTests.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.server.cli;
11+
12+
import org.elasticsearch.test.ESTestCase;
13+
14+
import java.util.List;
15+
16+
import static org.elasticsearch.server.cli.OverheadSystemMemoryInfo.SERVER_CLI_OVERHEAD;
17+
import static org.hamcrest.Matchers.is;
18+
19+
public class OverheadSystemMemoryInfoTests extends ESTestCase {
20+
21+
private static final long TRUE_SYSTEM_MEMORY = 1024 * 1024 * 1024L;
22+
23+
public void testNoOptions() {
24+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(List.of(), delegateSystemMemoryInfo());
25+
assertThat(memoryInfo.availableSystemMemory(), is(TRUE_SYSTEM_MEMORY - SERVER_CLI_OVERHEAD));
26+
}
27+
28+
public void testNoOverrides() {
29+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(List.of("-Da=b", "-Dx=y"), delegateSystemMemoryInfo());
30+
assertThat(memoryInfo.availableSystemMemory(), is(TRUE_SYSTEM_MEMORY - SERVER_CLI_OVERHEAD));
31+
}
32+
33+
public void testValidSingleOverride() {
34+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(
35+
List.of("-Des.total_memory_overhead_bytes=50000"),
36+
delegateSystemMemoryInfo()
37+
);
38+
assertThat(memoryInfo.availableSystemMemory(), is(TRUE_SYSTEM_MEMORY - 50000));
39+
}
40+
41+
public void testValidOverrideInList() {
42+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(
43+
List.of("-Da=b", "-Des.total_memory_overhead_bytes=50000", "-Dx=y"),
44+
delegateSystemMemoryInfo()
45+
);
46+
assertThat(memoryInfo.availableSystemMemory(), is(TRUE_SYSTEM_MEMORY - 50000));
47+
}
48+
49+
public void testMultipleValidOverridesInList() {
50+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(
51+
List.of("-Des.total_memory_overhead_bytes=50000", "-Da=b", "-Des.total_memory_overhead_bytes=100000", "-Dx=y"),
52+
delegateSystemMemoryInfo()
53+
);
54+
assertThat(memoryInfo.availableSystemMemory(), is(TRUE_SYSTEM_MEMORY - 100000));
55+
}
56+
57+
public void testNegativeOverride() {
58+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(
59+
List.of("-Da=b", "-Des.total_memory_overhead_bytes=-123", "-Dx=y"),
60+
delegateSystemMemoryInfo()
61+
);
62+
try {
63+
memoryInfo.availableSystemMemory();
64+
fail("expected to fail");
65+
} catch (IllegalArgumentException e) {
66+
assertThat(e.getMessage(), is("Negative bytes size specified in [-Des.total_memory_overhead_bytes=-123]"));
67+
}
68+
}
69+
70+
public void testUnparsableOverride() {
71+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(
72+
List.of("-Da=b", "-Des.total_memory_overhead_bytes=invalid", "-Dx=y"),
73+
delegateSystemMemoryInfo()
74+
);
75+
try {
76+
memoryInfo.availableSystemMemory();
77+
fail("expected to fail");
78+
} catch (IllegalArgumentException e) {
79+
assertThat(e.getMessage(), is("Unable to parse number of bytes from [-Des.total_memory_overhead_bytes=invalid]"));
80+
}
81+
}
82+
83+
private static SystemMemoryInfo delegateSystemMemoryInfo() {
84+
return () -> TRUE_SYSTEM_MEMORY;
85+
}
86+
}

‎distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/OverridableSystemMemoryInfoTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public void testNegativeOverride() {
6464
memoryInfo.availableSystemMemory();
6565
fail("expected to fail");
6666
} catch (IllegalArgumentException e) {
67-
assertThat(e.getMessage(), is("Negative memory size specified in [-Des.total_memory_bytes=-123]"));
67+
assertThat(e.getMessage(), is("Negative bytes size specified in [-Des.total_memory_bytes=-123]"));
6868
}
6969
}
7070

‎docs/changelog/137776.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 137776
2+
summary: Enable CCS tests for subqueries
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎docs/changelog/138467.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
pr: 138467
2+
summary: |
3+
Fixed a bug where dash-prefixed expressions were not consistently excluded during index resolution.
4+
This impacted both specific index names and wildcard patterns (example: `-index, -logs-*`).
5+
area: Security
6+
type: bug
7+
issues:
8+
- 64752
9+
- 83435

0 commit comments

Comments
 (0)