Skip to content

Commit e4ce906

Browse files
davidkyledavidkyle
authored
Merge branch 'main' into ccs-many-it
2 parents 06a5c31 + 7f01282 commit e4ce906

File tree

696 files changed

+23011
-11619
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

696 files changed

+23011
-11619
lines changed

‎benchmarks/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ dependencies {
4848
api(project(':x-pack:plugin:esql'))
4949
api(project(':x-pack:plugin:esql:compute'))
5050
api(project(':x-pack:plugin:mapper-exponential-histogram'))
51+
api(project(':x-pack:plugin:logsdb'))
5152
implementation project(path: ':libs:native')
5253
implementation project(path: ':libs:simdvec')
5354
implementation project(path: ':libs:exponential-histogram')

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@ public class QueryPlanningBenchmark {
7171
}
7272

7373
private PlanTelemetry telemetry;
74-
private EsqlParser defaultParser;
7574
private Analyzer manyFieldsAnalyzer;
7675
private LogicalPlanOptimizer defaultOptimizer;
7776
private Configuration config;
@@ -111,7 +110,6 @@ public void setup() {
111110
TransportVersion minimumVersion = TransportVersion.current();
112111

113112
telemetry = new PlanTelemetry(functionRegistry);
114-
defaultParser = new EsqlParser();
115113
manyFieldsAnalyzer = new Analyzer(
116114
new AnalyzerContext(
117115
config,
@@ -128,14 +126,14 @@ public void setup() {
128126
}
129127

130128
private LogicalPlan plan(EsqlParser parser, Analyzer analyzer, LogicalPlanOptimizer optimizer, String query) {
131-
var parsed = parser.createStatement(query, new QueryParams(), telemetry);
129+
var parsed = parser.parseQuery(query, new QueryParams(), telemetry);
132130
var analyzed = analyzer.analyze(parsed);
133131
var optimized = optimizer.optimize(analyzed);
134132
return optimized;
135133
}
136134

137135
@Benchmark
138136
public void manyFields(Blackhole blackhole) {
139-
blackhole.consume(plan(defaultParser, manyFieldsAnalyzer, defaultOptimizer, "FROM test | LIMIT 10"));
137+
blackhole.consume(plan(EsqlParser.INSTANCE, manyFieldsAnalyzer, defaultOptimizer, "FROM test | LIMIT 10"));
140138
}
141139
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/indices/common/MappingParsingBenchmark.java‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.benchmark.indices.common;
11+
12+
import org.elasticsearch.TransportVersion;
13+
import org.elasticsearch.cluster.ClusterModule;
14+
import org.elasticsearch.cluster.metadata.IndexMetadata;
15+
import org.elasticsearch.common.compress.CompressedXContent;
16+
import org.elasticsearch.common.logging.LogConfigurator;
17+
import org.elasticsearch.common.settings.IndexScopedSettings;
18+
import org.elasticsearch.common.settings.Setting;
19+
import org.elasticsearch.common.settings.Settings;
20+
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
21+
import org.elasticsearch.index.IndexSettings;
22+
import org.elasticsearch.index.IndexVersion;
23+
import org.elasticsearch.index.analysis.IndexAnalyzers;
24+
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
25+
import org.elasticsearch.index.mapper.MapperMetrics;
26+
import org.elasticsearch.index.mapper.MapperRegistry;
27+
import org.elasticsearch.index.mapper.MapperService;
28+
import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
29+
import org.elasticsearch.index.similarity.SimilarityService;
30+
import org.elasticsearch.indices.IndicesModule;
31+
import org.elasticsearch.script.Script;
32+
import org.elasticsearch.script.ScriptCompiler;
33+
import org.elasticsearch.script.ScriptContext;
34+
import org.elasticsearch.xcontent.NamedXContentRegistry;
35+
import org.elasticsearch.xcontent.XContentParserConfiguration;
36+
import org.elasticsearch.xpack.logsdb.LogsDBPlugin;
37+
import org.openjdk.jmh.annotations.Benchmark;
38+
import org.openjdk.jmh.annotations.BenchmarkMode;
39+
import org.openjdk.jmh.annotations.Fork;
40+
import org.openjdk.jmh.annotations.Measurement;
41+
import org.openjdk.jmh.annotations.Mode;
42+
import org.openjdk.jmh.annotations.OutputTimeUnit;
43+
import org.openjdk.jmh.annotations.Param;
44+
import org.openjdk.jmh.annotations.Scope;
45+
import org.openjdk.jmh.annotations.Setup;
46+
import org.openjdk.jmh.annotations.State;
47+
import org.openjdk.jmh.annotations.Warmup;
48+
49+
import java.io.IOException;
50+
import java.util.ArrayList;
51+
import java.util.HashSet;
52+
import java.util.List;
53+
import java.util.Map;
54+
import java.util.Random;
55+
import java.util.Set;
56+
import java.util.concurrent.TimeUnit;
57+
58+
@Fork(value = 1)
59+
@Warmup(iterations = 2)
60+
@Measurement(iterations = 5)
61+
@BenchmarkMode(Mode.AverageTime)
62+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
63+
@State(Scope.Benchmark)
64+
public class MappingParsingBenchmark {
65+
static {
66+
// For Elasticsearch900Lucene101Codec:
67+
LogConfigurator.loadLog4jPlugins();
68+
LogConfigurator.configureESLogging();
69+
LogConfigurator.setNodeName("test");
70+
}
71+
72+
private static final String MAPPING = """
73+
{
74+
"_doc": {
75+
"dynamic": false,
76+
"properties": {
77+
"@timestamp": {
78+
"type": "date"
79+
},
80+
"host": {
81+
"type": "object",
82+
"properties": {
83+
"name": {
84+
"type": "keyword"
85+
}
86+
}
87+
},
88+
"message": {
89+
"type": "pattern_text"
90+
}
91+
}
92+
}
93+
}
94+
\s""";
95+
96+
@Param("1024")
97+
private int numIndices;
98+
99+
private List<MapperService> mapperServices;
100+
private CompressedXContent compressedMapping;
101+
102+
private Random random = new Random();
103+
private static final String CHARS = "abcdefghijklmnopqrstuvwxyz1234567890";
104+
105+
private String randomIndexName() {
106+
StringBuilder b = new StringBuilder();
107+
for (int i = 0; i < 10; i++) {
108+
b.append(CHARS.charAt(random.nextInt(CHARS.length())));
109+
}
110+
return b.toString();
111+
}
112+
113+
@Setup
114+
public void setUp() throws IOException {
115+
Settings settings = Settings.builder()
116+
.put("index.number_of_replicas", 0)
117+
.put("index.number_of_shards", 1)
118+
.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
119+
.put("index.mode", "logsdb")
120+
.put("index.logsdb.sort_on_host_name", true)
121+
.put("index.logsdb.sort_on_message_template", true)
122+
.build();
123+
124+
LogsDBPlugin logsDBPlugin = new LogsDBPlugin(settings);
125+
126+
Set<Setting<?>> definedSettings = new HashSet<>(IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
127+
definedSettings.addAll(logsDBPlugin.getSettings().stream().filter(Setting::hasIndexScope).toList());
128+
IndexScopedSettings indexScopedSettings = new IndexScopedSettings(Settings.EMPTY, definedSettings);
129+
130+
mapperServices = new ArrayList<>(numIndices);
131+
for (int i = 0; i < numIndices; i++) {
132+
IndexMetadata meta = IndexMetadata.builder(randomIndexName()).settings(settings).build();
133+
IndexSettings indexSettings = new IndexSettings(meta, settings, indexScopedSettings);
134+
MapperRegistry mapperRegistry = new IndicesModule(List.of(logsDBPlugin)).getMapperRegistry();
135+
SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
136+
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP);
137+
MapperService mapperService = new MapperService(
138+
() -> TransportVersion.current(),
139+
indexSettings,
140+
IndexAnalyzers.of(Map.of()),
141+
XContentParserConfiguration.EMPTY.withRegistry(new NamedXContentRegistry(ClusterModule.getNamedXWriteables()))
142+
.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE),
143+
similarityService,
144+
mapperRegistry,
145+
() -> {
146+
throw new UnsupportedOperationException();
147+
},
148+
new ProvidedIdFieldMapper(() -> true),
149+
new ScriptCompiler() {
150+
@Override
151+
public <T> T compile(Script script, ScriptContext<T> scriptContext) {
152+
throw new UnsupportedOperationException();
153+
}
154+
},
155+
bitsetFilterCache::getBitSetProducer,
156+
MapperMetrics.NOOP
157+
);
158+
159+
mapperServices.add(mapperService);
160+
}
161+
162+
compressedMapping = new CompressedXContent(MAPPING);
163+
}
164+
165+
@Benchmark
166+
public void mappingParsingBenchmark() {
167+
for (MapperService service : mapperServices) {
168+
service.merge("_doc", compressedMapping, MapperService.MergeReason.MAPPING_UPDATE);
169+
}
170+
}
171+
}

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmArgumentParsingSystemMemoryInfo.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.server.cli;
11+
12+
import java.util.List;
13+
14+
public abstract class JvmArgumentParsingSystemMemoryInfo implements SystemMemoryInfo {
15+
private final List<String> userDefinedJvmOptions;
16+
17+
public JvmArgumentParsingSystemMemoryInfo(List<String> userDefinedJvmOptions) {
18+
this.userDefinedJvmOptions = userDefinedJvmOptions;
19+
}
20+
21+
protected long getBytesFromSystemProperty(String systemProperty, long defaultValue) {
22+
return userDefinedJvmOptions.stream()
23+
.filter(option -> option.startsWith("-D" + systemProperty + "="))
24+
.map(totalMemoryOverheadBytesOption -> {
25+
try {
26+
long bytes = Long.parseLong(totalMemoryOverheadBytesOption.split("=", 2)[1]);
27+
if (bytes < 0) {
28+
throw new IllegalArgumentException("Negative bytes size specified in [" + totalMemoryOverheadBytesOption + "]");
29+
}
30+
return bytes;
31+
} catch (NumberFormatException e) {
32+
throw new IllegalArgumentException("Unable to parse number of bytes from [" + totalMemoryOverheadBytesOption + "]", e);
33+
}
34+
})
35+
.reduce((previous, current) -> current) // this is effectively findLast(), so that ES_JAVA_OPTS overrides jvm.options
36+
.orElse(defaultValue);
37+
}
38+
}

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,10 @@ private List<String> jvmOptions(
142142
}
143143

144144
final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
145-
final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(substitutedJvmOptions, new DefaultSystemMemoryInfo());
145+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(
146+
substitutedJvmOptions,
147+
new OverridableSystemMemoryInfo(substitutedJvmOptions, new DefaultSystemMemoryInfo())
148+
);
146149
substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(args.nodeSettings(), memoryInfo, substitutedJvmOptions));
147150
final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions, args.nodeSettings());
148151
final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions(args.nodeSettings(), cliSysprops);

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/OverheadSystemMemoryInfo.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.server.cli;
11+
12+
import java.util.List;
13+
14+
/**
15+
* A {@link SystemMemoryInfo} implementation that reduces the reported available system memory by a set amount. This is intended to account
16+
* for overhead cost of other bundled Elasticsearch processes, such as the CLI tool launcher. By default, this is
17+
* {@link org.elasticsearch.server.cli.OverheadSystemMemoryInfo#SERVER_CLI_OVERHEAD } but can be overridden via the
18+
* {@code es.total_memory_overhead_bytes} system property.
19+
*/
20+
public class OverheadSystemMemoryInfo extends JvmArgumentParsingSystemMemoryInfo {
21+
static final long SERVER_CLI_OVERHEAD = 100 * 1024L * 1024L;
22+
23+
private final SystemMemoryInfo delegate;
24+
25+
public OverheadSystemMemoryInfo(List<String> userDefinedJvmOptions, SystemMemoryInfo delegate) {
26+
super(userDefinedJvmOptions);
27+
this.delegate = delegate;
28+
}
29+
30+
@Override
31+
public long availableSystemMemory() {
32+
long overheadBytes = getBytesFromSystemProperty("es.total_memory_overhead_bytes", SERVER_CLI_OVERHEAD);
33+
return delegate.availableSystemMemory() - overheadBytes;
34+
}
35+
}

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/OverridableSystemMemoryInfo.java‎

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -17,33 +17,17 @@
1717
* has been specified using the {@code es.total_memory_bytes} system property, or
1818
* else returns the value provided by a fallback provider.
1919
*/
20-
public final class OverridableSystemMemoryInfo implements SystemMemoryInfo {
20+
public final class OverridableSystemMemoryInfo extends JvmArgumentParsingSystemMemoryInfo {
2121

22-
private final List<String> userDefinedJvmOptions;
2322
private final SystemMemoryInfo fallbackSystemMemoryInfo;
2423

2524
public OverridableSystemMemoryInfo(final List<String> userDefinedJvmOptions, SystemMemoryInfo fallbackSystemMemoryInfo) {
26-
this.userDefinedJvmOptions = Objects.requireNonNull(userDefinedJvmOptions);
25+
super(userDefinedJvmOptions);
2726
this.fallbackSystemMemoryInfo = Objects.requireNonNull(fallbackSystemMemoryInfo);
2827
}
2928

3029
@Override
3130
public long availableSystemMemory() {
32-
33-
return userDefinedJvmOptions.stream()
34-
.filter(option -> option.startsWith("-Des.total_memory_bytes="))
35-
.map(totalMemoryBytesOption -> {
36-
try {
37-
long bytes = Long.parseLong(totalMemoryBytesOption.split("=", 2)[1]);
38-
if (bytes < 0) {
39-
throw new IllegalArgumentException("Negative memory size specified in [" + totalMemoryBytesOption + "]");
40-
}
41-
return bytes;
42-
} catch (NumberFormatException e) {
43-
throw new IllegalArgumentException("Unable to parse number of bytes from [" + totalMemoryBytesOption + "]", e);
44-
}
45-
})
46-
.reduce((previous, current) -> current) // this is effectively findLast(), so that ES_JAVA_OPTS overrides jvm.options
47-
.orElse(fallbackSystemMemoryInfo.availableSystemMemory());
31+
return getBytesFromSystemProperty("es.total_memory_bytes", fallbackSystemMemoryInfo.availableSystemMemory());
4832
}
4933
}

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,8 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
118118
return;
119119
}
120120

121+
// Call the GC to try and free up as much heap as we can since we don't intend to do much if any more allocation after this
122+
System.gc();
121123
// we are running in the foreground, so wait for the server to exit
122124
int exitCode = server.waitFor();
123125
onExit(exitCode);

0 commit comments

Comments
 (0)