Skip to content

Commit 108bcb0

Browse files
liustveliustve
authored
Configure EMF and CompactLog Exporters for Lambda Environment (#1222)
*Description of changes:* Java version of these PRs: - aws-observability/aws-otel-python-instrumentation#442 - aws-observability/aws-otel-python-instrumentation#437 This PR introduces Console EMF exporter and Compact Console logs exporter and enabling them for Lambda environments, allowing all forms of telemetry to be emitted directly to CloudWatch for users who've installed the Application Signals lambda layer. To enable the Compact Console logs exporter, users must set the following environment variables in their Lambda: - `OTEL_LOGS_EXPORTER`: `console` **required** To enable the Console EMF exporter, users must set the following environment variables in their Lambda: - `OTEL_METRICS_EXPORTER`: `awsemf` **required** - `OTEL_EXPORTER_OTLP_LOGS_HEADERS`: `x-aws-log-group=your-log-group,x-aws-log-stream=your-log-stream,x-aws-metric-namespace` **optional** Enabling the console exporters for EMF and logs will store the user's OTel metrics and logs directly in their Lambda log group. **Note 1:** if users would like to store their EMF logs to a separate log group, they **MUST*** set this configuration `OTEL_EXPORTER_OTLP_LOGS_HEADERS`: `x-aws-log-group=your-log-group,x-aws-log-stream=your-log-stream` **Note 2:** `x-aws-metric-namespace` will use `default` if it is not set **Testing**: - Added unit tests to validate EMF and console log record exporter configuration scenarios, including parameterized tests for both valid configurations and invalid configurations. The tests ensure the EMF and console log exporters are correctly enabled only when all required environment variables are properly configured. Manual end to end testing with a custom built Lambda layer using the sample app configured with the following environment variables in Lambda environment: <img width="763" height="235" alt="image" src="https://github.com/user-attachments/assets/2098a0a1-ea81-4999-89d5-4759f0a7822d" /> EMF translated OTel metrics and OTel logs: OTel log: <img width="1579" height="593" alt="image" src="https://github.com/user-attachments/assets/56451d44-d349-4cf3-9846-131057bd9e26" /> EMF log <img width="1578" height="482" alt="image" src="https://github.com/user-attachments/assets/91267f20-4928-4230-9b36-ace99b7a9eff" /> By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
1 parent b60ba73 commit 108bcb0

File tree

9 files changed

+888
-54
lines changed

9 files changed

+888
-54
lines changed

‎CHANGELOG.md‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,17 @@ If your change does not need a CHANGELOG entry, add the "skip changelog" label t
1313

1414
## Unreleased
1515

16+
### Enhancements
17+
18+
- Configure EMF and CompactLog Exporters for Lambda Environment
19+
([#1222](https://github.com/aws-observability/aws-otel-java-instrumentation/pull/1222))
20+
- feat: [Java] EMF Exporter Implementation
21+
([#1209](https://github.com/aws-observability/aws-otel-java-instrumentation/pull/1209))
22+
1623
## v2.20.0 - 2025-10-29
1724

1825
### Enhancements
1926

20-
- Add CloudWatch EMF metrics exporter with auto instrumentation configuration
21-
([#1209](https://github.com/aws-observability/aws-otel-java-instrumentation/pull/1209))
2227
- Support X-Ray Trace Id extraction from Lambda Context object, and respect user-configured OTEL_PROPAGATORS in AWS Lamdba instrumentation
2328
([#1191](https://github.com/aws-observability/aws-otel-java-instrumentation/pull/1191)) ([#1218](https://github.com/aws-observability/aws-otel-java-instrumentation/pull/1218))
2429
- Adaptive Sampling improvements: Ensure propagation of sampling rule across services and AWS accounts. Remove unnecessary B3 propagator.

‎awsagentprovider/src/main/java/software/amazon/opentelemetry/javaagent/providers/AwsApplicationSignalsCustomizerProvider.java‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,10 @@
6565
import java.util.logging.Level;
6666
import java.util.logging.Logger;
6767
import javax.annotation.concurrent.Immutable;
68+
import software.amazon.opentelemetry.javaagent.providers.exporter.aws.logs.CompactConsoleLogRecordExporter;
6869
import software.amazon.opentelemetry.javaagent.providers.exporter.aws.metrics.AwsCloudWatchEmfExporter;
6970
import software.amazon.opentelemetry.javaagent.providers.exporter.aws.metrics.ConsoleEmfExporter;
70-
import software.amazon.opentelemetry.javaagent.providers.exporter.otlp.aws.logs.OtlpAwsLogsExporterBuilder;
71+
import software.amazon.opentelemetry.javaagent.providers.exporter.otlp.aws.logs.OtlpAwsLogRecordExporterBuilder;
7172
import software.amazon.opentelemetry.javaagent.providers.exporter.otlp.aws.traces.OtlpAwsSpanExporterBuilder;
7273

7374
/**
@@ -91,7 +92,10 @@ public final class AwsApplicationSignalsCustomizerProvider
9192
// https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
9293
static final String AWS_REGION = "aws.region";
9394
static final String AWS_DEFAULT_REGION = "aws.default.region";
95+
// TODO: We should clean up and get rid of using AWS_LAMBDA_FUNCTION_NAME and default to
96+
// upstream config property implementation.
9497
static final String AWS_LAMBDA_FUNCTION_NAME_CONFIG = "AWS_LAMBDA_FUNCTION_NAME";
98+
static final String AWS_LAMBDA_FUNCTION_NAME_PROP_CONFIG = "aws.lambda.function.name";
9599
static final String LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT =
96100
"LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT";
97101

@@ -181,6 +185,10 @@ public void customize(AutoConfigurationCustomizer autoConfiguration) {
181185
autoConfiguration.addMetricExporterCustomizer(this::customizeMetricExporter);
182186
}
183187

188+
static boolean isLambdaEnvironment(ConfigProperties props) {
189+
return props.getString(AWS_LAMBDA_FUNCTION_NAME_PROP_CONFIG) != null;
190+
}
191+
184192
private static Optional<String> getAwsRegionFromConfig(ConfigProperties configProps) {
185193
String region = configProps.getString(AWS_REGION);
186194
if (region != null) {
@@ -515,7 +523,7 @@ LogRecordExporter customizeLogsExporter(
515523
configProps.getString(OTEL_EXPORTER_OTLP_COMPRESSION_CONFIG, "none"));
516524

517525
try {
518-
return OtlpAwsLogsExporterBuilder.create(
526+
return OtlpAwsLogRecordExporterBuilder.create(
519527
(OtlpHttpLogRecordExporter) logsExporter,
520528
configProps.getString(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT))
521529
.setCompression(compression)
@@ -528,34 +536,41 @@ LogRecordExporter customizeLogsExporter(
528536
e);
529537
}
530538
}
539+
String logsExporterConfig = configProps.getString(OTEL_LOGS_EXPORTER);
540+
541+
if (isLambdaEnvironment(configProps)
542+
&& logsExporterConfig != null
543+
&& logsExporterConfig.equals("console")) {
544+
return new CompactConsoleLogRecordExporter();
545+
}
531546

532547
return logsExporter;
533548
}
534549

535550
MetricExporter customizeMetricExporter(
536551
MetricExporter metricExporter, ConfigProperties configProps) {
552+
537553
if (isEmfExporterEnabled) {
538554
Map<String, String> headers =
539555
AwsApplicationSignalsConfigUtils.parseOtlpHeaders(
540556
configProps.getString(OTEL_EXPORTER_OTLP_LOGS_HEADERS));
541557
Optional<String> awsRegion = getAwsRegionFromConfig(configProps);
558+
String namespace = headers.get(AWS_EMF_METRICS_NAMESPACE);
542559

543560
if (awsRegion.isPresent()) {
544-
String namespace = headers.get(AWS_EMF_METRICS_NAMESPACE);
545-
546561
if (headers.containsKey(AWS_OTLP_LOGS_GROUP_HEADER)
547562
&& headers.containsKey(AWS_OTLP_LOGS_STREAM_HEADER)) {
548563
String logGroup = headers.get(AWS_OTLP_LOGS_GROUP_HEADER);
549564
String logStream = headers.get(AWS_OTLP_LOGS_STREAM_HEADER);
550565
return new AwsCloudWatchEmfExporter(namespace, logGroup, logStream, awsRegion.get());
551566
}
552-
if (isLambdaEnvironment()) {
567+
568+
if (isLambdaEnvironment(configProps)) {
553569
return new ConsoleEmfExporter(namespace);
554570
}
555571
logger.warning(
556572
String.format(
557-
"Improper EMF Exporter configuration: Please configure the environment variable %s to have values for %s, %s, and %s",
558-
OTEL_EXPORTER_OTLP_LOGS_HEADERS,
573+
"Improper EMF Exporter configuration: Please configure the environment variable OTEL_EXPORTER_OTLP_LOGS_HEADERS to have values for %s, %s, and %s",
559574
AWS_OTLP_LOGS_GROUP_HEADER,
560575
AWS_OTLP_LOGS_STREAM_HEADER,
561576
AWS_EMF_METRICS_NAMESPACE));

‎awsagentprovider/src/main/java/software/amazon/opentelemetry/javaagent/providers/exporter/aws/logs/CompactConsoleLogRecordExporter.java‎

Lines changed: 277 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,277 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.opentelemetry.javaagent.providers.exporter.aws.logs;
17+
18+
/*
19+
* Copyright The OpenTelemetry Authors
20+
* SPDX-License-Identifier: Apache-2.0
21+
*
22+
* Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
23+
*/
24+
25+
import com.fasterxml.jackson.annotation.JsonProperty;
26+
import com.fasterxml.jackson.databind.ObjectMapper;
27+
import com.fasterxml.jackson.databind.SerializationFeature;
28+
import io.opentelemetry.exporter.internal.otlp.IncubatingUtil;
29+
import io.opentelemetry.sdk.common.CompletableResultCode;
30+
import io.opentelemetry.sdk.common.InstrumentationScopeInfo;
31+
import io.opentelemetry.sdk.logs.data.LogRecordData;
32+
import io.opentelemetry.sdk.logs.export.LogRecordExporter;
33+
import io.opentelemetry.sdk.resources.Resource;
34+
import java.io.PrintStream;
35+
import java.time.Instant;
36+
import java.time.ZoneOffset;
37+
import java.time.format.DateTimeFormatter;
38+
import java.util.Collection;
39+
import java.util.HashMap;
40+
import java.util.Map;
41+
import java.util.concurrent.TimeUnit;
42+
import java.util.concurrent.atomic.AtomicBoolean;
43+
44+
/**
45+
* A {@link LogRecordExporter} that prints {@link LogRecordData} to standard out based on upstream's
46+
* implementation of SystemOutLogRecordExporter, see: <a
47+
* href="https://github.com/open-telemetry/opentelemetry-java/blob/5ab0a65675e5a06d13b293a758ef495d797e6d04/exporters/logging/src/main/java/io/opentelemetry/exporter/logging/SystemOutLogRecordExporter.java#L66">...</a>
48+
*/
49+
@SuppressWarnings("SystemOut")
50+
public class CompactConsoleLogRecordExporter implements LogRecordExporter {
51+
private static final DateTimeFormatter ISO_FORMAT = DateTimeFormatter.ISO_INSTANT;
52+
private static final ObjectMapper MAPPER =
53+
new ObjectMapper().disable(SerializationFeature.INDENT_OUTPUT);
54+
private final AtomicBoolean isShutdown = new AtomicBoolean();
55+
private final PrintStream printStream;
56+
57+
public CompactConsoleLogRecordExporter() {
58+
this(System.out);
59+
}
60+
61+
public CompactConsoleLogRecordExporter(PrintStream printStream) {
62+
this.printStream = printStream;
63+
}
64+
65+
@Override
66+
public CompletableResultCode export(Collection<LogRecordData> logs) {
67+
if (isShutdown.get()) {
68+
return CompletableResultCode.ofFailure();
69+
}
70+
71+
for (LogRecordData log : logs) {
72+
this.printStream.println(this.toCompactJson(log));
73+
this.printStream.flush();
74+
}
75+
return CompletableResultCode.ofSuccess();
76+
}
77+
78+
@Override
79+
public CompletableResultCode flush() {
80+
this.printStream.flush();
81+
return CompletableResultCode.ofSuccess();
82+
}
83+
84+
@Override
85+
public CompletableResultCode shutdown() {
86+
if (!this.isShutdown.compareAndSet(false, true)) {
87+
this.printStream.println("Calling shutdown() multiple times.");
88+
}
89+
return CompletableResultCode.ofSuccess();
90+
}
91+
92+
@Override
93+
public String toString() {
94+
return "CompactConsoleLogRecordExporter{}";
95+
}
96+
97+
/**
98+
* Converts OpenTelemetry log data to compact JSON format. OTel Java's SystemOutLogRecordExporter
99+
* uses a concise text format, this implementation outputs a compact JSON representation based on
100+
* OTel JavaScript's _exportInfo: <a
101+
* href="https://github.com/open-telemetry/opentelemetry-js/blob/09bf31eb966bab627e76a6c5c05c6e51ccd2f387/experimental/packages/sdk-logs/src/export/ConsoleLogRecordExporter.ts#L58">...</a>
102+
*
103+
* <p>Example output:
104+
*
105+
* <pre>
106+
* {"body":"This is a test log","severityNumber":9,"severityText":"INFO","attributes":{},"droppedAttributes":0,"timestamp":"2025-09-30T22:37:56.724Z","observedTimestamp":"2025-09-30T22:37:56.724Z","traceId":"","spanId":"","traceFlags":0,"resource":{}}
107+
* </pre>
108+
*
109+
* @param log the log record data to convert
110+
* @return compact JSON string representation of the log record
111+
*/
112+
private String toCompactJson(LogRecordData log) {
113+
LogRecordDataTemplate template = LogRecordDataTemplate.parse(log);
114+
115+
try {
116+
return MAPPER.writeValueAsString(template);
117+
} catch (Exception e) {
118+
this.printStream.println("Error serializing log record: " + e.getMessage());
119+
return "{}";
120+
}
121+
}
122+
123+
/** Data object that structures OTel log record data for JSON serialization. */
124+
@SuppressWarnings("unused")
125+
private static final class LogRecordDataTemplate {
126+
@JsonProperty("resource")
127+
private final ResourceTemplate resourceTemplate;
128+
129+
@JsonProperty("body")
130+
private final String body;
131+
132+
@JsonProperty("severityNumber")
133+
private final int severityNumber;
134+
135+
@JsonProperty("severityText")
136+
private final String severityText;
137+
138+
@JsonProperty("attributes")
139+
private final Map<String, String> attributes;
140+
141+
@JsonProperty("droppedAttributes")
142+
private final int droppedAttributes;
143+
144+
@JsonProperty("timestamp")
145+
private final String timestamp;
146+
147+
@JsonProperty("observedTimestamp")
148+
private final String observedTimestamp;
149+
150+
@JsonProperty("traceId")
151+
private final String traceId;
152+
153+
@JsonProperty("spanId")
154+
private final String spanId;
155+
156+
@JsonProperty("traceFlags")
157+
private final int traceFlags;
158+
159+
@JsonProperty("instrumentationScope")
160+
private final InstrumentationScopeTemplate instrumentationScope;
161+
162+
private LogRecordDataTemplate(
163+
String body,
164+
int severityNumber,
165+
String severityText,
166+
Map<String, String> attributes,
167+
int droppedAttributes,
168+
String timestamp,
169+
String observedTimestamp,
170+
String traceId,
171+
String spanId,
172+
int traceFlags,
173+
ResourceTemplate resourceTemplate,
174+
InstrumentationScopeTemplate instrumentationScope) {
175+
this.resourceTemplate = resourceTemplate;
176+
this.body = body;
177+
this.severityNumber = severityNumber;
178+
this.severityText = severityText;
179+
this.attributes = attributes;
180+
this.droppedAttributes = droppedAttributes;
181+
this.timestamp = timestamp;
182+
this.observedTimestamp = observedTimestamp;
183+
this.traceId = traceId;
184+
this.spanId = spanId;
185+
this.traceFlags = traceFlags;
186+
this.instrumentationScope = instrumentationScope;
187+
}
188+
189+
private static LogRecordDataTemplate parse(LogRecordData log) {
190+
// https://github.com/open-telemetry/opentelemetry-java/blob/48684d6d33048030b133b4f6479d45afddcdc313/exporters/otlp/common/src/main/java/io/opentelemetry/exporter/internal/otlp/logs/LogMarshaler.java#L59
191+
Map<String, String> attributes = new HashMap<>();
192+
log.getAttributes()
193+
.forEach((key, value) -> attributes.put(key.getKey(), String.valueOf(value)));
194+
195+
int attributeSize =
196+
IncubatingUtil.isExtendedLogRecordData(log)
197+
? IncubatingUtil.extendedAttributesSize(log)
198+
: log.getAttributes().size();
199+
200+
return new LogRecordDataTemplate(
201+
log.getBodyValue() != null ? log.getBodyValue().asString() : null,
202+
log.getSeverity().getSeverityNumber(),
203+
log.getSeverity().name(),
204+
attributes,
205+
log.getTotalAttributeCount() - attributeSize,
206+
formatTimestamp(log.getTimestampEpochNanos()),
207+
formatTimestamp(log.getObservedTimestampEpochNanos()),
208+
log.getSpanContext().isValid() ? log.getSpanContext().getTraceId() : "",
209+
log.getSpanContext().isValid() ? log.getSpanContext().getSpanId() : "",
210+
log.getSpanContext().getTraceFlags().asByte(),
211+
log.getResource() != null
212+
? ResourceTemplate.parse(log.getResource())
213+
: new ResourceTemplate(new HashMap<>(), ""),
214+
log.getInstrumentationScopeInfo() != null
215+
? InstrumentationScopeTemplate.parse(log.getInstrumentationScopeInfo())
216+
: new InstrumentationScopeTemplate("", "", ""));
217+
}
218+
219+
private static String formatTimestamp(long nanos) {
220+
return nanos != 0
221+
? ISO_FORMAT.format(
222+
Instant.ofEpochMilli(TimeUnit.NANOSECONDS.toMillis(nanos)).atZone(ZoneOffset.UTC))
223+
: null;
224+
}
225+
}
226+
227+
@SuppressWarnings("unused")
228+
private static final class ResourceTemplate {
229+
@JsonProperty("attributes")
230+
private final Map<String, String> attributes;
231+
232+
@JsonProperty("schemaUrl")
233+
private final String schemaUrl;
234+
235+
private ResourceTemplate(Map<String, String> attributes, String schemaUrl) {
236+
this.attributes = attributes;
237+
this.schemaUrl = schemaUrl != null ? schemaUrl : "";
238+
}
239+
240+
private static ResourceTemplate parse(Resource resource) {
241+
Map<String, String> attributes = new HashMap<>();
242+
if (resource == null) {
243+
return new ResourceTemplate(attributes, "");
244+
}
245+
resource
246+
.getAttributes()
247+
.forEach((key, value) -> attributes.put(key.getKey(), String.valueOf(value)));
248+
return new ResourceTemplate(attributes, resource.getSchemaUrl());
249+
}
250+
}
251+
252+
@SuppressWarnings("unused")
253+
private static final class InstrumentationScopeTemplate {
254+
@JsonProperty("name")
255+
private final String name;
256+
257+
@JsonProperty("version")
258+
private final String version;
259+
260+
@JsonProperty("schemaUrl")
261+
private final String schemaUrl;
262+
263+
private InstrumentationScopeTemplate(String name, String version, String schemaUrl) {
264+
this.name = name != null ? name : "";
265+
this.version = version != null ? version : "";
266+
this.schemaUrl = schemaUrl != null ? schemaUrl : "";
267+
}
268+
269+
private static InstrumentationScopeTemplate parse(InstrumentationScopeInfo scope) {
270+
if (scope == null) {
271+
return new InstrumentationScopeTemplate("", "", "");
272+
}
273+
return new InstrumentationScopeTemplate(
274+
scope.getName(), scope.getVersion(), scope.getSchemaUrl());
275+
}
276+
}
277+
}

0 commit comments

Comments
 (0)