Skip to content

Commit d3d49f9

Browse files
simplynaveen20simplynaveen20
authored
Adding client telemetry (Azure#16822)
* client telemetry changes * build fix * adding incremental change for client telemetry * adding incremental change for client telemetry * adding incremental change for client telemetry * incremental check in * Incremental work * Incremental work * incremental work * adding setAutoResize on cpu and memory * adding useragent in serialization * Adding test file * adding suprression for reportpayload equals method * performance turning and resolving comments * resolving comments and merge with latest * resolving comments * fixing build error * resolving build error * moving before and after from simple to emulator for telemetry * merge with master and changing database field name in report payload
1 parent f4a5de0 commit d3d49f9

34 files changed

+1586
-112
lines changed

‎eng/code-quality-reports/src/main/resources/spotbugs/spotbugs-exclude.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1997,6 +1997,13 @@
19971997
<Bug pattern="EQ_UNUSUAL"/>
19981998
</Match>
19991999

2000+
<!-- False positive, equals method is implemented correctly by using equality of member variables -->
2001+
<Match>
2002+
<Class name="com.azure.cosmos.implementation.clientTelemetry.ReportPayload"/>
2003+
<Method name="equals"/>
2004+
<Bug pattern="EQ_UNUSUAL"/>
2005+
</Match>
2006+
20002007
<!-- Bug: https://github.com/Azure/azure-sdk-for-java/issues/9106 -->
20012008
<Match>
20022009
<Class name="com.azure.cosmos.implementation.PathsHelper"/>

‎eng/versioning/external_dependencies.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,7 @@ org.codehaus.mojo:properties-maven-plugin;1.0.0
249249
org.codehaus.mojo:xml-maven-plugin;1.0
250250
org.eclipse.jetty:jetty-maven-plugin;9.4.33.v20201020
251251
org.eclipse.m2e:lifecycle-mapping;1.0.0
252+
org.hdrhistogram:HdrHistogram;2.1.12
252253
org.jacoco:jacoco-maven-plugin;0.8.5
253254
org.jacoco:org.jacoco.agent;0.8.5
254255
org.mockito:mockito-core;3.3.3
@@ -280,6 +281,7 @@ cosmos_io.dropwizard.metrics:metrics-graphite;4.1.0
280281
cosmos_io.dropwizard.metrics:metrics-jvm;4.1.0
281282
cosmos_org.mockito:mockito-core;1.10.19
282283
cosmos_org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir;1.1.0
284+
cosmos_org.hdrhistogram:HdrHistogram;2.1.12
283285

284286
# sdk\core\azure-core-serializer-avro-jackson\pom.xml
285287
# This dependency is needed since Jackson Avro uses an older dependency on Apache Avro which is another library.

‎sdk/cosmos/azure-cosmos/pom.xml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,12 @@ Licensed under the MIT License.
104104
<scope>test</scope>
105105
<version>4.2</version> <!-- {x-version-update;org.apache.commons:commons-collections4;external_dependency} -->
106106
</dependency>
107-
107+
<!-- https://mvnrepository.com/artifact/org.hdrhistogram/HdrHistogram -->
108+
<dependency>
109+
<groupId>org.hdrhistogram</groupId>
110+
<artifactId>HdrHistogram</artifactId>
111+
<version>2.1.12</version> <!-- {x-version-update;cosmos_org.hdrhistogram:HdrHistogram;external_dependency} -->
112+
</dependency>
108113
<dependency>
109114
<groupId>org.apache.commons</groupId>
110115
<artifactId>commons-text</artifactId>
@@ -201,7 +206,6 @@ Licensed under the MIT License.
201206
<skip>true</skip>
202207
</configuration>
203208
</plugin>
204-
205209
<plugin>
206210
<groupId>org.apache.maven.plugins</groupId>
207211
<artifactId>maven-surefire-plugin</artifactId>
@@ -232,6 +236,7 @@ Licensed under the MIT License.
232236
<include>io.dropwizard.metrics:metrics-core:[4.1.0]</include> <!-- {x-include-update;cosmos_io.dropwizard.metrics:metrics-core;external_dependency} -->
233237
<include>io.micrometer:micrometer-core:[1.5.6]</include> <!-- {x-include-update;io.micrometer:micrometer-core;external_dependency} -->
234238
<include>org.slf4j:slf4j-api:[1.7.30]</include> <!-- {x-include-update;org.slf4j:slf4j-api;external_dependency} -->
239+
<include>org.hdrhistogram:HdrHistogram:[2.1.12]</include> <!-- {x-include-update;cosmos_org.hdrhistogram:HdrHistogram;external_dependency} -->
235240
</includes>
236241
</bannedDependencies>
237242
</rules>

‎sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/BridgeInternal.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
package com.azure.cosmos;
55

6+
import com.azure.cosmos.implementation.AsyncDocumentClient;
67
import com.azure.cosmos.implementation.Configs;
78
import com.azure.cosmos.implementation.Constants;
89
import com.azure.cosmos.implementation.CosmosError;
@@ -25,6 +26,7 @@
2526
import com.azure.cosmos.implementation.SerializationDiagnosticsContext;
2627
import com.azure.cosmos.implementation.ServiceUnavailableException;
2728
import com.azure.cosmos.implementation.StoredProcedureResponse;
29+
import com.azure.cosmos.implementation.TracerProvider;
2830
import com.azure.cosmos.implementation.Warning;
2931
import com.azure.cosmos.implementation.directconnectivity.StoreResponse;
3032
import com.azure.cosmos.implementation.directconnectivity.StoreResult;
@@ -33,6 +35,7 @@
3335
import com.azure.cosmos.implementation.query.QueryInfo;
3436
import com.azure.cosmos.implementation.query.metrics.ClientSideMetrics;
3537
import com.azure.cosmos.implementation.routing.PartitionKeyInternal;
38+
import com.azure.cosmos.models.CosmosItemRequestOptions;
3639
import com.azure.cosmos.models.CosmosItemResponse;
3740
import com.azure.cosmos.models.CosmosStoredProcedureProperties;
3841
import com.azure.cosmos.models.FeedResponse;
@@ -70,6 +73,26 @@ public static CosmosDiagnostics createCosmosDiagnostics(DiagnosticsClientContext
7073
return new CosmosDiagnostics(diagnosticsClientContext);
7174
}
7275

76+
@Warning(value = INTERNAL_USE_ONLY_WARNING)
77+
public static Set<URI> getRegionsContacted(CosmosDiagnostics cosmosDiagnostics) {
78+
return cosmosDiagnostics.clientSideRequestStatistics().getRegionsContacted();
79+
}
80+
81+
@Warning(value = INTERNAL_USE_ONLY_WARNING)
82+
public static AsyncDocumentClient getContextClient(CosmosAsyncClient cosmosAsyncClient) {
83+
return cosmosAsyncClient.getContextClient();
84+
}
85+
86+
@Warning(value = INTERNAL_USE_ONLY_WARNING)
87+
public static String getServiceEndpoint(CosmosAsyncClient cosmosAsyncClient) {
88+
return cosmosAsyncClient.getServiceEndpoint();
89+
}
90+
91+
@Warning(value = INTERNAL_USE_ONLY_WARNING)
92+
public static boolean isClientTelemetryEnabled(CosmosAsyncClient cosmosAsyncClient) {
93+
return cosmosAsyncClient.isClientTelemetryEnabled();
94+
}
95+
7396
@Warning(value = INTERNAL_USE_ONLY_WARNING)
7497
public static Document documentFromObject(Object document, ObjectMapper mapper) {
7598
return Document.fromObject(document, mapper);
@@ -578,6 +601,11 @@ public static CosmosDatabase createCosmosDatabase(String id, CosmosClient client
578601
return new CosmosDatabase(id, client, database);
579602
}
580603

604+
@Warning(value = INTERNAL_USE_ONLY_WARNING)
605+
public static TracerProvider getTracerProvider(CosmosAsyncClient client) {
606+
return client.getTracerProvider();
607+
}
608+
581609
@Warning(value = INTERNAL_USE_ONLY_WARNING)
582610
public static CosmosUser createCosmosUser(CosmosAsyncUser asyncUser, CosmosDatabase database, String id) {
583611
return new CosmosUser(asyncUser, database, id);

‎sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/ClientSideRequestStatistics.java‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
package com.azure.cosmos;
44

55
import com.azure.cosmos.implementation.DiagnosticsClientContext;
6+
import com.azure.cosmos.implementation.DiagnosticsInstantSerializer;
67
import com.azure.cosmos.implementation.HttpConstants;
78
import com.azure.cosmos.implementation.MetadataDiagnosticsContext;
89
import com.azure.cosmos.implementation.OperationType;
@@ -12,16 +13,15 @@
1213
import com.azure.cosmos.implementation.RxDocumentServiceRequest;
1314
import com.azure.cosmos.implementation.SerializationDiagnosticsContext;
1415
import com.azure.cosmos.implementation.Utils;
15-
import com.azure.cosmos.implementation.DiagnosticsInstantSerializer;
16-
import com.azure.cosmos.implementation.cpu.CpuMonitor;
16+
import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
17+
import com.azure.cosmos.implementation.cpu.CpuMemoryMonitor;
1718
import com.azure.cosmos.implementation.directconnectivity.DirectBridgeInternal;
1819
import com.azure.cosmos.implementation.directconnectivity.StoreResponse;
1920
import com.azure.cosmos.implementation.directconnectivity.StoreResult;
2021
import com.fasterxml.jackson.core.JsonGenerator;
2122
import com.fasterxml.jackson.databind.SerializerProvider;
2223
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
2324
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
24-
import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
2525

2626
import java.io.IOException;
2727
import java.net.URI;
@@ -120,18 +120,25 @@ void recordGatewayResponse(
120120
CosmosException exception) {
121121
Instant responseTime = Instant.now();
122122
connectionMode = ConnectionMode.GATEWAY;
123+
124+
123125
synchronized (this) {
124126
if (responseTime.isAfter(this.requestEndTimeUTC)) {
125127
this.requestEndTimeUTC = responseTime;
126128
}
127129

128-
if (rxDocumentServiceRequest != null
129-
&& rxDocumentServiceRequest.requestContext != null
130-
&& rxDocumentServiceRequest.requestContext.retryContext != null) {
131-
rxDocumentServiceRequest.requestContext.retryContext.retryEndTime = Instant.now();
132-
this.retryContext = new RetryContext(rxDocumentServiceRequest.requestContext.retryContext);
130+
URI locationEndPoint = null;
131+
if (rxDocumentServiceRequest != null && rxDocumentServiceRequest.requestContext != null) {
132+
locationEndPoint = rxDocumentServiceRequest.requestContext.locationEndpointToRoute;
133+
if (rxDocumentServiceRequest.requestContext.retryContext != null) {
134+
rxDocumentServiceRequest.requestContext.retryContext.retryEndTime = Instant.now();
135+
this.retryContext = new RetryContext(rxDocumentServiceRequest.requestContext.retryContext);
136+
}
133137
}
134138

139+
if (locationEndPoint != null) {
140+
this.regionsContacted.add(locationEndPoint);
141+
}
135142
this.gatewayStatistics = new GatewayStatistics();
136143
if (rxDocumentServiceRequest != null) {
137144
this.gatewayStatistics.operationType = rxDocumentServiceRequest.getOperationType();
@@ -312,7 +319,7 @@ public void serialize(
312319
systemInformation.availableMemory = (maxMemory - (totalMemory - freeMemory)) + " KB";
313320

314321
// TODO: other system related info also can be captured using a similar approach
315-
systemInformation.systemCpuLoad = CpuMonitor.getCpuLoad().toString();
322+
systemInformation.systemCpuLoad = CpuMemoryMonitor.getCpuLoad().toString();
316323
generator.writeObjectField("systemInformation", systemInformation);
317324
} catch (Exception e) {
318325
// Error while evaluating system information, do nothing
@@ -372,4 +379,4 @@ public RequestTimeline getRequestTimeline() {
372379
return requestTimeline;
373380
}
374381
}
375-
}
382+
}

‎sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncClient.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ public final class CosmosAsyncClient implements Closeable {
5959
private final TokenCredential tokenCredential;
6060
private final boolean sessionCapturingOverride;
6161
private final boolean enableTransportClientSharing;
62+
private final boolean clientTelemetryEnabled;
6263
private final TracerProvider tracerProvider;
6364
private final boolean contentResponseOnWriteEnabled;
6465
private static final Tracer TRACER;
@@ -85,6 +86,7 @@ public final class CosmosAsyncClient implements Closeable {
8586
this.tokenCredential = builder.getTokenCredential();
8687
this.sessionCapturingOverride = builder.isSessionCapturingOverrideEnabled();
8788
this.enableTransportClientSharing = builder.isConnectionSharingAcrossClientsEnabled();
89+
this.clientTelemetryEnabled = builder.isClientTelemetryEnabled();
8890
this.contentResponseOnWriteEnabled = builder.isContentResponseOnWriteEnabled();
8991
this.tracerProvider = new TracerProvider(TRACER);
9092
this.asyncDocumentClient = new AsyncDocumentClient.Builder()
@@ -207,6 +209,10 @@ boolean isContentResponseOnWriteEnabled() {
207209
return contentResponseOnWriteEnabled;
208210
}
209211

212+
boolean isClientTelemetryEnabled() {
213+
return clientTelemetryEnabled;
214+
}
215+
210216
/**
211217
* CREATE a Database if it does not already exist on the service.
212218
* <p>

‎sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncContainer.java‎

Lines changed: 39 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@
1010
import com.azure.cosmos.implementation.InternalObjectNode;
1111
import com.azure.cosmos.implementation.ItemDeserializer;
1212
import com.azure.cosmos.implementation.Offer;
13+
import com.azure.cosmos.implementation.OperationType;
1314
import com.azure.cosmos.implementation.Paths;
1415
import com.azure.cosmos.implementation.RequestOptions;
16+
import com.azure.cosmos.implementation.ResourceType;
1517
import com.azure.cosmos.implementation.TracerProvider;
1618
import com.azure.cosmos.implementation.Utils;
1719
import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
@@ -35,7 +37,6 @@
3537
import com.azure.cosmos.util.Beta;
3638
import com.azure.cosmos.util.CosmosPagedFlux;
3739
import com.azure.cosmos.util.UtilBridgeInternal;
38-
3940
import reactor.core.publisher.Flux;
4041
import reactor.core.publisher.Mono;
4142

@@ -262,8 +263,12 @@ private <T> Mono<CosmosItemResponse<T>> createItemInternal(T item, CosmosItemReq
262263
return database.getClient().getTracerProvider().traceEnabledCosmosItemResponsePublisher(responseMono,
263264
context,
264265
this.createItemSpanName,
266+
getId(),
265267
database.getId(),
266-
database.getClient().getServiceEndpoint());
268+
database.getClient(),
269+
ModelBridgeInternal.getConsistencyLevel(options),
270+
OperationType.Create,
271+
ResourceType.Document);
267272
}
268273

269274
private <T> Mono<CosmosItemResponse<T>> createItemInternal(T item, CosmosItemRequestOptions options) {
@@ -361,9 +366,8 @@ <T> CosmosPagedFlux<T> readAllItems(Class<T> classType) {
361366
*/
362367
<T> CosmosPagedFlux<T> readAllItems(CosmosQueryRequestOptions options, Class<T> classType) {
363368
return UtilBridgeInternal.createCosmosPagedFlux(pagedFluxOptions -> {
364-
pagedFluxOptions.setTracerInformation(this.getDatabase().getClient().getTracerProvider(),
365-
this.readAllItemsSpanName,
366-
this.getDatabase().getClient().getServiceEndpoint(), database.getId());
369+
pagedFluxOptions.setTracerAndTelemetryInformation(this.readAllItemsSpanName, database.getId(),
370+
this.getId(), OperationType.ReadFeed, ResourceType.Document, this.getDatabase().getClient());
367371
setContinuationTokenAndMaxItemCount(pagedFluxOptions, options);
368372
return getDatabase().getDocClientWrapper().readDocuments(getLink(), options).map(
369373
response -> prepareFeedResponse(response, classType));
@@ -457,8 +461,8 @@ <T> Function<CosmosPagedFluxOptions, Flux<FeedResponse<T>>> queryItemsInternalFu
457461
SqlQuerySpec sqlQuerySpec, CosmosQueryRequestOptions cosmosQueryRequestOptions, Class<T> classType) {
458462
Function<CosmosPagedFluxOptions, Flux<FeedResponse<T>>> pagedFluxOptionsFluxFunction = (pagedFluxOptions -> {
459463
String spanName = this.queryItemsSpanName;
460-
pagedFluxOptions.setTracerInformation(this.getDatabase().getClient().getTracerProvider(), spanName,
461-
this.getDatabase().getClient().getServiceEndpoint(), database.getId());
464+
pagedFluxOptions.setTracerAndTelemetryInformation(spanName, database.getId(),
465+
this.getId(), OperationType.Query, ResourceType.Document, this.getDatabase().getClient());
462466
setContinuationTokenAndMaxItemCount(pagedFluxOptions, cosmosQueryRequestOptions);
463467

464468
return getDatabase().getDocClientWrapper()
@@ -701,18 +705,17 @@ public <T> CosmosPagedFlux<T> readAllItems(
701705
* @param partitionKey the partition key value of the documents that need to be read
702706
* @param options the feed options.
703707
* @param classType the class type.
704-
* @return a {@link CosmosPagedFlux} containing one or several feed response pages
708+
* @return a {@link CosmosPagedFlux} containing one or several feed response pages
705709
* of the read Cosmos items or an error.
706710
*/
707711
public <T> CosmosPagedFlux<T> readAllItems(
708712
PartitionKey partitionKey,
709713
CosmosQueryRequestOptions options,
710714
Class<T> classType) {
711-
715+
712716
return UtilBridgeInternal.createCosmosPagedFlux(pagedFluxOptions -> {
713-
pagedFluxOptions.setTracerInformation(this.getDatabase().getClient().getTracerProvider(),
714-
this.readAllItemsSpanName,
715-
this.getDatabase().getClient().getServiceEndpoint(), database.getId());
717+
pagedFluxOptions.setTracerAndTelemetryInformation(this.readAllItemsSpanName, database.getId(),
718+
this.getId(), OperationType.ReadFeed, ResourceType.Document, this.getDatabase().getClient());
716719
setContinuationTokenAndMaxItemCount(pagedFluxOptions, options);
717720
return getDatabase()
718721
.getDocClientWrapper()
@@ -857,6 +860,7 @@ public CosmosPagedFlux<CosmosConflictProperties> readAllConflicts(CosmosQueryReq
857860
pagedFluxOptions.setTracerInformation(this.getDatabase().getClient().getTracerProvider(),
858861
this.readAllConflictsSpanName,
859862
this.getDatabase().getClient().getServiceEndpoint(), database.getId());
863+
860864
setContinuationTokenAndMaxItemCount(pagedFluxOptions, requestOptions);
861865
return database.getDocClientWrapper().readConflicts(getLink(), requestOptions)
862866
.map(response -> BridgeInternal.createFeedResponse(
@@ -961,8 +965,12 @@ private Mono<CosmosItemResponse<Object>> deleteItemInternal(
961965
return database.getClient().getTracerProvider().traceEnabledCosmosItemResponsePublisher(responseMono,
962966
context,
963967
this.deleteItemSpanName,
968+
this.getId(),
964969
database.getId(),
965-
database.getClient().getServiceEndpoint());
970+
database.getClient(),
971+
requestOptions.getConsistencyLevel(),
972+
OperationType.Delete,
973+
ResourceType.Document);
966974
}
967975

968976
private <T> Mono<CosmosItemResponse<T>> replaceItemInternal(
@@ -977,7 +985,14 @@ private <T> Mono<CosmosItemResponse<T>> replaceItemInternal(
977985
.map(response -> ModelBridgeInternal.createCosmosAsyncItemResponse(response, itemType, getItemDeserializer()))
978986
.single();
979987
return database.getClient().getTracerProvider().traceEnabledCosmosItemResponsePublisher(responseMono,
980-
context, this.replaceItemSpanName, database.getId(), database.getClient().getServiceEndpoint());
988+
context,
989+
this.replaceItemSpanName,
990+
this.getId(),
991+
database.getId(),
992+
database.getClient(),
993+
ModelBridgeInternal.getConsistencyLevel(options),
994+
OperationType.Replace,
995+
ResourceType.Document);
981996
}
982997

983998
private <T> Mono<CosmosItemResponse<T>> upsertItemInternal(T item, CosmosItemRequestOptions options, Context context) {
@@ -992,8 +1007,12 @@ private <T> Mono<CosmosItemResponse<T>> upsertItemInternal(T item, CosmosItemReq
9921007
return database.getClient().getTracerProvider().traceEnabledCosmosItemResponsePublisher(responseMono,
9931008
context,
9941009
this.upsertItemSpanName,
1010+
this.getId(),
9951011
database.getId(),
996-
database.getClient().getServiceEndpoint());
1012+
database.getClient(),
1013+
ModelBridgeInternal.getConsistencyLevel(options),
1014+
OperationType.Upsert,
1015+
ResourceType.Document);
9971016
}
9981017

9991018
private <T> Mono<CosmosItemResponse<T>> readItemInternal(
@@ -1007,8 +1026,12 @@ private <T> Mono<CosmosItemResponse<T>> readItemInternal(
10071026
return database.getClient().getTracerProvider().traceEnabledCosmosItemResponsePublisher(responseMono,
10081027
context,
10091028
this.readItemSpanName,
1029+
this.getId(),
10101030
database.getId(),
1011-
database.getClient().getServiceEndpoint());
1031+
database.getClient(),
1032+
requestOptions.getConsistencyLevel(),
1033+
OperationType.Read,
1034+
ResourceType.Document);
10121035
}
10131036

10141037
Mono<CosmosContainerResponse> read(CosmosContainerRequestOptions options, Context context) {

0 commit comments

Comments
 (0)