Skip to content

Commit 3ed0e0d

Browse files
authored
improv(metrics): Add thread-safety to Metrics utility through request-scoped Metrics instance management (#2294)
* Add proxy to manage thread local metrics provider instances to achieve zero-lock thread-safety. * Refactor from inheritable thread local to tracking by trace id. Similar to log buffer. * Do not hard code JVM system property for trace id. * Fix PMD findings.
1 parent 4e83d8a commit 3ed0e0d

File tree

9 files changed

+536
-40
lines changed

9 files changed

+536
-40
lines changed

powertools-metrics/src/main/java/software/amazon/lambda/powertools/metrics/MetricsFactory.java

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,11 @@
1616

1717
import org.crac.Core;
1818
import org.crac.Resource;
19+
1920
import software.amazon.lambda.powertools.common.internal.ClassPreLoader;
2021
import software.amazon.lambda.powertools.common.internal.LambdaConstants;
2122
import software.amazon.lambda.powertools.common.internal.LambdaHandlerProcessor;
23+
import software.amazon.lambda.powertools.metrics.internal.RequestScopedMetricsProxy;
2224
import software.amazon.lambda.powertools.metrics.model.DimensionSet;
2325
import software.amazon.lambda.powertools.metrics.provider.EmfMetricsProvider;
2426
import software.amazon.lambda.powertools.metrics.provider.MetricsProvider;
@@ -28,7 +30,7 @@
2830
*/
2931
public final class MetricsFactory implements Resource {
3032
private static MetricsProvider provider = new EmfMetricsProvider();
31-
private static Metrics metrics;
33+
private static RequestScopedMetricsProxy metricsProxy;
3234

3335
// Dummy instance to register MetricsFactory with CRaC
3436
private static final MetricsFactory INSTANCE = new MetricsFactory();
@@ -44,23 +46,23 @@ public final class MetricsFactory implements Resource {
4446
* @return the singleton Metrics instance
4547
*/
4648
public static synchronized Metrics getMetricsInstance() {
47-
if (metrics == null) {
48-
metrics = provider.getMetricsInstance();
49+
if (metricsProxy == null) {
50+
metricsProxy = new RequestScopedMetricsProxy(provider);
4951

5052
// Apply default configuration from environment variables
5153
String envNamespace = System.getenv("POWERTOOLS_METRICS_NAMESPACE");
5254
if (envNamespace != null) {
53-
metrics.setNamespace(envNamespace);
55+
metricsProxy.setNamespace(envNamespace);
5456
}
5557

5658
// Only set Service dimension if it's not the default undefined value
5759
String serviceName = LambdaHandlerProcessor.serviceName();
5860
if (!LambdaConstants.SERVICE_UNDEFINED.equals(serviceName)) {
59-
metrics.setDefaultDimensions(DimensionSet.of("Service", serviceName));
61+
metricsProxy.setDefaultDimensions(DimensionSet.of("Service", serviceName));
6062
}
6163
}
6264

63-
return metrics;
65+
return metricsProxy;
6466
}
6567

6668
/**
@@ -73,8 +75,8 @@ public static synchronized void setMetricsProvider(MetricsProvider metricsProvid
7375
throw new IllegalArgumentException("Metrics provider cannot be null");
7476
}
7577
provider = metricsProvider;
76-
// Reset the metrics instance so it will be recreated with the new provider
77-
metrics = null;
78+
// Reset the metrics proxy so it will be recreated with the new provider
79+
metricsProxy = null;
7880
}
7981

8082
@Override
Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
/*
2+
* Copyright 2023 Amazon.com, Inc. or its affiliates.
3+
* Licensed under the Apache License, Version 2.0 (the
4+
* "License"); you may not use this file except in compliance
5+
* with the License. You may obtain a copy of the License at
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
* Unless required by applicable law or agreed to in writing, software
8+
* distributed under the License is distributed on an "AS IS" BASIS,
9+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
* See the License for the specific language governing permissions and
11+
* limitations under the License.
12+
*
13+
*/
14+
15+
package software.amazon.lambda.powertools.metrics.internal;
16+
17+
import java.time.Instant;
18+
import java.util.HashMap;
19+
import java.util.Optional;
20+
import java.util.concurrent.ConcurrentHashMap;
21+
import java.util.concurrent.ConcurrentMap;
22+
import java.util.concurrent.atomic.AtomicBoolean;
23+
import java.util.concurrent.atomic.AtomicReference;
24+
import java.util.function.Consumer;
25+
26+
import com.amazonaws.services.lambda.runtime.Context;
27+
28+
import software.amazon.lambda.powertools.common.internal.LambdaHandlerProcessor;
29+
import software.amazon.lambda.powertools.metrics.Metrics;
30+
import software.amazon.lambda.powertools.metrics.model.DimensionSet;
31+
import software.amazon.lambda.powertools.metrics.model.MetricResolution;
32+
import software.amazon.lambda.powertools.metrics.model.MetricUnit;
33+
import software.amazon.lambda.powertools.metrics.provider.MetricsProvider;
34+
35+
public class RequestScopedMetricsProxy implements Metrics {
36+
private static final String DEFAULT_TRACE_ID = "DEFAULT";
37+
private final ConcurrentMap<String, Metrics> metricsMap = new ConcurrentHashMap<>();
38+
private final MetricsProvider provider;
39+
private final AtomicReference<String> initialNamespace = new AtomicReference<>();
40+
private final AtomicReference<DimensionSet> initialDefaultDimensions = new AtomicReference<>();
41+
private final AtomicBoolean initialRaiseOnEmptyMetrics = new AtomicBoolean(false);
42+
43+
public RequestScopedMetricsProxy(MetricsProvider provider) {
44+
this.provider = provider;
45+
}
46+
47+
private String getTraceId() {
48+
return LambdaHandlerProcessor.getXrayTraceId().orElse(DEFAULT_TRACE_ID);
49+
}
50+
51+
private Metrics getOrCreateMetrics() {
52+
String traceId = getTraceId();
53+
return metricsMap.computeIfAbsent(traceId, key -> {
54+
Metrics metrics = provider.getMetricsInstance();
55+
String namespace = initialNamespace.get();
56+
if (namespace != null) {
57+
metrics.setNamespace(namespace);
58+
}
59+
DimensionSet dimensions = initialDefaultDimensions.get();
60+
if (dimensions != null) {
61+
metrics.setDefaultDimensions(dimensions);
62+
}
63+
metrics.setRaiseOnEmptyMetrics(initialRaiseOnEmptyMetrics.get());
64+
return metrics;
65+
});
66+
}
67+
68+
// Configuration methods - called by MetricsFactory and MetricsBuilder
69+
// These methods DO NOT eagerly create instances because they are typically called
70+
// outside the Lambda handler (e.g., during class initialization) potentially on a different thread.
71+
// We delay instance creation until the first operation that needs the metrics backend (e.g., addMetric).
72+
// See {@link software.amazon.lambda.powertools.metrics.MetricsFactory#getMetricsInstance()}
73+
// and {@link software.amazon.lambda.powertools.metrics.MetricsBuilder#build()}
74+
75+
@Override
76+
public void setNamespace(String namespace) {
77+
this.initialNamespace.set(namespace);
78+
Optional.ofNullable(metricsMap.get(getTraceId())).ifPresent(m -> m.setNamespace(namespace));
79+
}
80+
81+
@Override
82+
public void setDefaultDimensions(DimensionSet dimensionSet) {
83+
if (dimensionSet == null) {
84+
throw new IllegalArgumentException("DimensionSet cannot be null");
85+
}
86+
this.initialDefaultDimensions.set(dimensionSet);
87+
Optional.ofNullable(metricsMap.get(getTraceId())).ifPresent(m -> m.setDefaultDimensions(dimensionSet));
88+
}
89+
90+
@Override
91+
public void setRaiseOnEmptyMetrics(boolean raiseOnEmptyMetrics) {
92+
this.initialRaiseOnEmptyMetrics.set(raiseOnEmptyMetrics);
93+
Optional.ofNullable(metricsMap.get(getTraceId())).ifPresent(m -> m.setRaiseOnEmptyMetrics(raiseOnEmptyMetrics));
94+
}
95+
96+
@Override
97+
public DimensionSet getDefaultDimensions() {
98+
Metrics metrics = metricsMap.get(getTraceId());
99+
if (metrics != null) {
100+
return metrics.getDefaultDimensions();
101+
}
102+
DimensionSet dimensions = initialDefaultDimensions.get();
103+
return dimensions != null ? dimensions : DimensionSet.of(new HashMap<>());
104+
}
105+
106+
// Metrics operations - these eagerly create instances
107+
108+
@Override
109+
public void addMetric(String key, double value, MetricUnit unit, MetricResolution resolution) {
110+
getOrCreateMetrics().addMetric(key, value, unit, resolution);
111+
}
112+
113+
@Override
114+
public void addDimension(DimensionSet dimensionSet) {
115+
getOrCreateMetrics().addDimension(dimensionSet);
116+
}
117+
118+
@Override
119+
public void setTimestamp(Instant timestamp) {
120+
getOrCreateMetrics().setTimestamp(timestamp);
121+
}
122+
123+
@Override
124+
public void addMetadata(String key, Object value) {
125+
getOrCreateMetrics().addMetadata(key, value);
126+
}
127+
128+
@Override
129+
public void clearDefaultDimensions() {
130+
getOrCreateMetrics().clearDefaultDimensions();
131+
}
132+
133+
@Override
134+
public void flush() {
135+
// Always create instance to ensure validation and warnings are triggered. E.g. when raiseOnEmptyMetrics
136+
// is enabled.
137+
Metrics metrics = getOrCreateMetrics();
138+
metrics.flush();
139+
metricsMap.remove(getTraceId());
140+
}
141+
142+
@Override
143+
public void captureColdStartMetric(Context context, DimensionSet dimensions) {
144+
getOrCreateMetrics().captureColdStartMetric(context, dimensions);
145+
}
146+
147+
@Override
148+
public void captureColdStartMetric(DimensionSet dimensions) {
149+
getOrCreateMetrics().captureColdStartMetric(dimensions);
150+
}
151+
152+
@Override
153+
public void flushMetrics(Consumer<Metrics> metricsConsumer) {
154+
getOrCreateMetrics().flushMetrics(metricsConsumer);
155+
}
156+
}

powertools-metrics/src/test/java/software/amazon/lambda/powertools/metrics/ConfigurationPrecedenceTest.java

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@
3333
import com.fasterxml.jackson.databind.ObjectMapper;
3434

3535
import software.amazon.lambda.powertools.common.internal.LambdaHandlerProcessor;
36-
import software.amazon.lambda.powertools.metrics.model.MetricUnit;
3736
import software.amazon.lambda.powertools.common.stubs.TestLambdaContext;
37+
import software.amazon.lambda.powertools.metrics.model.MetricUnit;
3838

3939
/**
4040
* Tests to verify the hierarchy of precedence for configuration:
@@ -44,7 +44,7 @@
4444
*/
4545
class ConfigurationPrecedenceTest {
4646

47-
private final PrintStream standardOut = System.out;
47+
private static final PrintStream STANDARD_OUT = System.out;
4848
private final ByteArrayOutputStream outputStreamCaptor = new ByteArrayOutputStream();
4949
private final ObjectMapper objectMapper = new ObjectMapper();
5050

@@ -65,10 +65,10 @@ void setUp() throws Exception {
6565

6666
@AfterEach
6767
void tearDown() throws Exception {
68-
System.setOut(standardOut);
68+
System.setOut(STANDARD_OUT);
6969

7070
// Reset the singleton state between tests
71-
java.lang.reflect.Field field = MetricsFactory.class.getDeclaredField("metrics");
71+
java.lang.reflect.Field field = MetricsFactory.class.getDeclaredField("metricsProxy");
7272
field.setAccessible(true);
7373
field.set(null, null);
7474

@@ -183,7 +183,7 @@ void shouldUseDefaultsWhenNoConfiguration() throws Exception {
183183
assertThat(rootNode.has("Service")).isFalse();
184184
}
185185

186-
private static class HandlerWithMetricsAnnotation implements RequestHandler<Map<String, Object>, String> {
186+
private static final class HandlerWithMetricsAnnotation implements RequestHandler<Map<String, Object>, String> {
187187
@Override
188188
@FlushMetrics(namespace = "AnnotationNamespace", service = "AnnotationService")
189189
public String handleRequest(Map<String, Object> input, Context context) {
@@ -193,7 +193,8 @@ public String handleRequest(Map<String, Object> input, Context context) {
193193
}
194194
}
195195

196-
private static class HandlerWithDefaultMetricsAnnotation implements RequestHandler<Map<String, Object>, String> {
196+
private static final class HandlerWithDefaultMetricsAnnotation
197+
implements RequestHandler<Map<String, Object>, String> {
197198
@Override
198199
@FlushMetrics
199200
public String handleRequest(Map<String, Object> input, Context context) {

powertools-metrics/src/test/java/software/amazon/lambda/powertools/metrics/MetricsBuilderTest.java

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,15 @@
2727
import com.fasterxml.jackson.databind.JsonNode;
2828
import com.fasterxml.jackson.databind.ObjectMapper;
2929

30+
import software.amazon.lambda.powertools.metrics.internal.RequestScopedMetricsProxy;
3031
import software.amazon.lambda.powertools.metrics.model.DimensionSet;
3132
import software.amazon.lambda.powertools.metrics.model.MetricUnit;
3233
import software.amazon.lambda.powertools.metrics.provider.MetricsProvider;
33-
import software.amazon.lambda.powertools.metrics.testutils.TestMetrics;
3434
import software.amazon.lambda.powertools.metrics.testutils.TestMetricsProvider;
3535

3636
class MetricsBuilderTest {
3737

38-
private final PrintStream standardOut = System.out;
38+
private static final PrintStream STANDARD_OUT = System.out;
3939
private final ByteArrayOutputStream outputStreamCaptor = new ByteArrayOutputStream();
4040
private final ObjectMapper objectMapper = new ObjectMapper();
4141

@@ -46,10 +46,10 @@ void setUp() {
4646

4747
@AfterEach
4848
void tearDown() throws Exception {
49-
System.setOut(standardOut);
49+
System.setOut(STANDARD_OUT);
5050

5151
// Reset the singleton state between tests
52-
java.lang.reflect.Field field = MetricsFactory.class.getDeclaredField("metrics");
52+
java.lang.reflect.Field field = MetricsFactory.class.getDeclaredField("metricsProxy");
5353
field.setAccessible(true);
5454
field.set(null, null);
5555

@@ -151,7 +151,7 @@ void shouldBuildWithMultipleDefaultDimensions() throws Exception {
151151
}
152152

153153
@Test
154-
void shouldBuildWithCustomMetricsProvider() {
154+
void shouldBuildWithCustomMetricsProvider() throws Exception {
155155
// Given
156156
MetricsProvider testProvider = new TestMetricsProvider();
157157

@@ -161,7 +161,13 @@ void shouldBuildWithCustomMetricsProvider() {
161161
.build();
162162

163163
// Then
164-
assertThat(metrics).isInstanceOf(TestMetrics.class);
164+
assertThat(metrics)
165+
.isInstanceOf(RequestScopedMetricsProxy.class);
166+
167+
java.lang.reflect.Field providerField = metrics.getClass().getDeclaredField("provider");
168+
providerField.setAccessible(true);
169+
MetricsProvider actualProvider = (MetricsProvider) providerField.get(metrics);
170+
assertThat(actualProvider).isSameAs(testProvider);
165171
}
166172

167173
@Test

0 commit comments

Comments
 (0)