Skip to content

Commit 049524b

Browse files
FabianMeiswinkelannie-mac
FabianMeiswinkel
and
annie-mac
authored
MaxRetryCountTest+[BUG]404/1002CyclicRegionRetry+[BUG]449RetryTooAggressively +[BUG]MissingCrossRegionRetryForWritesWithServer410 Azure#37050) (Azure#37040)
* adding maxRetryCount tests for 404/1002, 500, 503, 408 * fixed 404/1002 cyclicRegionRetry * fixed 449 retry too aggressively * fixed missing cross region retry for write with server 410 * fixed missing cross region retry for server returned 503 for write operations --------- Co-authored-by: annie-mac <xinlian@microsoft.com>
1 parent 2cd4878 commit 049524b

File tree

11 files changed

+2949
-302
lines changed

11 files changed

+2949
-302
lines changed

‎sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionServerErrorResultInternal.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,8 @@ public CosmosException getInjectedServerError(RxDocumentServiceRequest request)
9090
responseHeaders.put(
9191
HttpConstants.HttpHeaders.RETRY_AFTER_IN_MILLISECONDS,
9292
String.valueOf(500));
93+
responseHeaders.put(WFConstants.BackendHeaders.SUB_STATUS,
94+
Integer.toString(HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE));
9395
cosmosException = new RequestRateTooLargeException(null, lsn, partitionKeyRangeId, responseHeaders);
9496

9597
break;
@@ -137,7 +139,8 @@ public CosmosException getInjectedServerError(RxDocumentServiceRequest request)
137139
case SERVICE_UNAVAILABLE:
138140
responseHeaders.put(WFConstants.BackendHeaders.SUB_STATUS,
139141
Integer.toString(HttpConstants.SubStatusCodes.SERVER_GENERATED_503));
140-
cosmosException = new ServiceUnavailableException(null, lsn, null, responseHeaders, HttpConstants.SubStatusCodes.SERVER_GENERATED_503);
142+
cosmosException =
143+
new ServiceUnavailableException(null, lsn, null, responseHeaders, HttpConstants.SubStatusCodes.SERVER_GENERATED_503);
141144
break;
142145

143146
case STALED_ADDRESSES_SERVER_GONE:

‎sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/FaultInjectionWithAvailabilityStrategyTests.java‎

Lines changed: 69 additions & 67 deletions
Large diffs are not rendered by default.

‎sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/MaxRetryCountTests.java‎

Lines changed: 2661 additions & 0 deletions
Large diffs are not rendered by default.

‎sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/ExcludedRegionWithFaultInjectionTests.java‎

Lines changed: 32 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1099,9 +1099,9 @@ public Object[][] regionExclusionWriteAfterCreateTestConfigs() {
10991099
this.chooseFirstRegion.apply(this.preferredRegions)
11001100
))
11011101
.withExpectedResultAfterMutation(new ExpectedResult(
1102-
HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
1103-
HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
1104-
this.chooseFirstRegion.apply(this.preferredRegions)
1102+
HttpConstants.StatusCodes.CREATED,
1103+
HttpConstants.SubStatusCodes.UNKNOWN,
1104+
this.chooseAllRegions.apply(this.preferredRegions)
11051105
)),
11061106
},
11071107
{
@@ -1159,9 +1159,9 @@ public Object[][] regionExclusionWriteAfterCreateTestConfigs() {
11591159
this.chooseFirstRegion.apply(this.preferredRegions)
11601160
))
11611161
.withExpectedResultAfterMutation(new ExpectedResult(
1162-
HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
1163-
HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
1164-
this.chooseFirstRegion.apply(this.preferredRegions)
1162+
HttpConstants.StatusCodes.OK,
1163+
HttpConstants.SubStatusCodes.UNKNOWN,
1164+
this.chooseAllRegions.apply(this.preferredRegions)
11651165
)),
11661166
},
11671167
{
@@ -1224,9 +1224,9 @@ public Object[][] regionExclusionWriteAfterCreateTestConfigs() {
12241224
this.chooseFirstRegion.apply(this.preferredRegions)
12251225
))
12261226
.withExpectedResultAfterMutation(new ExpectedResult(
1227-
HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
1228-
HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
1229-
this.chooseFirstRegion.apply(this.preferredRegions)
1227+
HttpConstants.StatusCodes.OK,
1228+
HttpConstants.SubStatusCodes.UNKNOWN,
1229+
this.chooseAllRegions.apply(this.preferredRegions)
12301230
)),
12311231
},
12321232
{
@@ -1266,9 +1266,9 @@ public Object[][] regionExclusionWriteAfterCreateTestConfigs() {
12661266
this.chooseFirstRegion.apply(this.preferredRegions)
12671267
))
12681268
.withExpectedResultAfterMutation(new ExpectedResult(
1269-
HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
1270-
HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
1271-
this.chooseFirstRegion.apply(this.preferredRegions)
1269+
HttpConstants.StatusCodes.CREATED,
1270+
HttpConstants.SubStatusCodes.UNKNOWN,
1271+
this.chooseAllRegions.apply(this.preferredRegions)
12721272
)),
12731273
},
12741274
{
@@ -1308,9 +1308,9 @@ public Object[][] regionExclusionWriteAfterCreateTestConfigs() {
13081308
this.chooseFirstRegion.apply(this.preferredRegions)
13091309
))
13101310
.withExpectedResultAfterMutation(new ExpectedResult(
1311-
HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
1312-
HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
1313-
this.chooseFirstRegion.apply(this.preferredRegions)
1311+
HttpConstants.StatusCodes.NO_CONTENT,
1312+
HttpConstants.SubStatusCodes.UNKNOWN,
1313+
this.chooseAllRegions.apply(this.preferredRegions)
13141314
)),
13151315
},
13161316
{
@@ -1350,9 +1350,9 @@ public Object[][] regionExclusionWriteAfterCreateTestConfigs() {
13501350
this.chooseFirstRegion.apply(this.preferredRegions)
13511351
))
13521352
.withExpectedResultAfterMutation(new ExpectedResult(
1353-
HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
1354-
HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
1355-
this.chooseFirstRegion.apply(this.preferredRegions)
1353+
HttpConstants.StatusCodes.OK,
1354+
HttpConstants.SubStatusCodes.UNKNOWN,
1355+
this.chooseAllRegions.apply(this.preferredRegions)
13561356
)),
13571357
},
13581358
{
@@ -1668,9 +1668,10 @@ public Object[][] regionExclusionBatchTestConfigs() {
16681668
this.chooseLastRegion.apply(this.preferredRegions)
16691669
))
16701670
.withExpectedResultAfterMutation(new ExpectedResult(
1671-
HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
1672-
HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
1673-
this.chooseFirstRegion.apply(this.preferredRegions)
1671+
HttpConstants.StatusCodes.OK,
1672+
HttpConstants.SubStatusCodes.UNKNOWN,
1673+
2,
1674+
this.chooseFirstTwoRegions.apply(this.preferredRegions)
16741675
))
16751676
},
16761677
{
@@ -1689,9 +1690,10 @@ public Object[][] regionExclusionBatchTestConfigs() {
16891690
this.chooseLastRegion.apply(this.preferredRegions)
16901691
))
16911692
.withExpectedResultAfterMutation(new ExpectedResult(
1692-
HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
1693-
HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
1694-
this.chooseFirstRegion.apply(this.preferredRegions)
1693+
HttpConstants.StatusCodes.OK,
1694+
HttpConstants.SubStatusCodes.UNKNOWN,
1695+
2,
1696+
this.chooseFirstTwoRegions.apply(this.preferredRegions)
16951697
))
16961698
},
16971699
{
@@ -2035,9 +2037,9 @@ public Object[][] regionExclusionBulkTestConfigs() {
20352037
this.chooseLastRegion.apply(this.preferredRegions)
20362038
))
20372039
.withExpectedResultAfterMutation(new ExpectedResult(
2038-
HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
2039-
HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
2040-
this.chooseFirstRegion.apply(this.preferredRegions)
2040+
HttpConstants.StatusCodes.OK,
2041+
HttpConstants.SubStatusCodes.UNKNOWN,
2042+
this.chooseFirstTwoRegions.apply(this.preferredRegions)
20412043
))
20422044
},
20432045
{
@@ -2055,9 +2057,9 @@ public Object[][] regionExclusionBulkTestConfigs() {
20552057
this.chooseLastRegion.apply(this.preferredRegions)
20562058
))
20572059
.withExpectedResultAfterMutation(new ExpectedResult(
2058-
HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
2059-
HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
2060-
this.chooseFirstRegion.apply(this.preferredRegions)
2060+
HttpConstants.StatusCodes.OK,
2061+
HttpConstants.SubStatusCodes.UNKNOWN,
2062+
this.chooseFirstTwoRegions.apply(this.preferredRegions)
20612063
))
20622064
},
20632065
{

‎sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleOnDirectTests.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ public static Object[][] faultInjectionServerErrorResponseProvider() {
124124
{ OperationType.Read, FaultInjectionOperationType.READ_ITEM, FaultInjectionServerErrorType.GONE, true, 410, HttpConstants.SubStatusCodes.SERVER_GENERATED_410 },
125125
{ OperationType.Read, FaultInjectionOperationType.READ_ITEM, FaultInjectionServerErrorType.INTERNAL_SERVER_ERROR, false, 500, 0 },
126126
{ OperationType.Read, FaultInjectionOperationType.READ_ITEM, FaultInjectionServerErrorType.RETRY_WITH, true, 449, 0 },
127-
{ OperationType.Read, FaultInjectionOperationType.READ_ITEM, FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, 0 },
127+
{ OperationType.Read, FaultInjectionOperationType.READ_ITEM, FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE },
128128
{ OperationType.Read, FaultInjectionOperationType.READ_ITEM, FaultInjectionServerErrorType.READ_SESSION_NOT_AVAILABLE, true, 404, 1002 },
129129
{ OperationType.Read, FaultInjectionOperationType.READ_ITEM, FaultInjectionServerErrorType.TIMEOUT, true, 410, HttpConstants.SubStatusCodes.SERVER_GENERATED_408 }, // for server return 408, SDK will wrap into 410/21010
130130
{ OperationType.Read, FaultInjectionOperationType.READ_ITEM, FaultInjectionServerErrorType.PARTITION_IS_MIGRATING, true, 410, 1008 },
@@ -133,7 +133,7 @@ public static Object[][] faultInjectionServerErrorResponseProvider() {
133133
{ OperationType.ReadFeed, FaultInjectionOperationType.READ_FEED_ITEM, FaultInjectionServerErrorType.GONE, true, 410, HttpConstants.SubStatusCodes.SERVER_GENERATED_410 },
134134
{ OperationType.ReadFeed, FaultInjectionOperationType.READ_FEED_ITEM, FaultInjectionServerErrorType.INTERNAL_SERVER_ERROR, false, 500, 0 },
135135
{ OperationType.ReadFeed, FaultInjectionOperationType.READ_FEED_ITEM, FaultInjectionServerErrorType.RETRY_WITH, true, 449, 0 },
136-
{ OperationType.ReadFeed, FaultInjectionOperationType.READ_FEED_ITEM, FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, 0 },
136+
{ OperationType.ReadFeed, FaultInjectionOperationType.READ_FEED_ITEM, FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE },
137137
{ OperationType.ReadFeed, FaultInjectionOperationType.READ_FEED_ITEM, FaultInjectionServerErrorType.READ_SESSION_NOT_AVAILABLE, true, 404, 1002 },
138138
{ OperationType.ReadFeed, FaultInjectionOperationType.READ_FEED_ITEM, FaultInjectionServerErrorType.TIMEOUT, true, 410, HttpConstants.SubStatusCodes.SERVER_GENERATED_408 }, // for server return 408, SDK will wrap into 410/21010
139139
{ OperationType.ReadFeed, FaultInjectionOperationType.READ_FEED_ITEM, FaultInjectionServerErrorType.PARTITION_IS_MIGRATING, true, 410, 1008 },
@@ -142,7 +142,7 @@ public static Object[][] faultInjectionServerErrorResponseProvider() {
142142
{ OperationType.Create, FaultInjectionOperationType.CREATE_ITEM, FaultInjectionServerErrorType.GONE, true, 410, HttpConstants.SubStatusCodes.SERVER_GENERATED_410 },
143143
{ OperationType.Create, FaultInjectionOperationType.CREATE_ITEM, FaultInjectionServerErrorType.INTERNAL_SERVER_ERROR, false, 500, 0 },
144144
{ OperationType.Create, FaultInjectionOperationType.CREATE_ITEM, FaultInjectionServerErrorType.RETRY_WITH, true, 449, 0 },
145-
{ OperationType.Create, FaultInjectionOperationType.CREATE_ITEM, FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, 0 },
145+
{ OperationType.Create, FaultInjectionOperationType.CREATE_ITEM, FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE },
146146
{ OperationType.Create, FaultInjectionOperationType.CREATE_ITEM, FaultInjectionServerErrorType.TIMEOUT, false, 410, HttpConstants.SubStatusCodes.SERVER_GENERATED_408 }, // for server return 408, SDK will wrap into 410/21010
147147
{ OperationType.Create, FaultInjectionOperationType.CREATE_ITEM, FaultInjectionServerErrorType.PARTITION_IS_MIGRATING, true, 410, 1008 },
148148
{ OperationType.Create, FaultInjectionOperationType.CREATE_ITEM, FaultInjectionServerErrorType.PARTITION_IS_SPLITTING, true, 410, 1007 },
@@ -220,7 +220,7 @@ public void faultInjectionServerErrorRuleTests_OperationType(OperationType opera
220220
cosmosDiagnostics,
221221
operationType,
222222
HttpConstants.StatusCodes.TOO_MANY_REQUESTS,
223-
HttpConstants.SubStatusCodes.UNKNOWN,
223+
HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE,
224224
tooManyRequestsRuleId,
225225
true);
226226
} else {
@@ -494,7 +494,7 @@ public void faultInjectionServerErrorRuleTests_Partition() throws JsonProcessing
494494
cosmosDiagnostics,
495495
OperationType.Read,
496496
HttpConstants.StatusCodes.TOO_MANY_REQUESTS,
497-
HttpConstants.SubStatusCodes.UNKNOWN,
497+
HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE,
498498
feedRangeRuleId,
499499
true
500500
);

‎sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleOnGatewayTests.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ public static Object[][] faultInjectionServerErrorResponseProvider() {
122122
// faultInjectionServerError, will SDK retry, errorStatusCode, errorSubStatusCode
123123
{ FaultInjectionServerErrorType.INTERNAL_SERVER_ERROR, false, 500, 0 },
124124
{ FaultInjectionServerErrorType.RETRY_WITH, false, 449, 0 },
125-
{ FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, 0 },
125+
{ FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE },
126126
{ FaultInjectionServerErrorType.READ_SESSION_NOT_AVAILABLE, true, 404, 1002 },
127127
{ FaultInjectionServerErrorType.SERVICE_UNAVAILABLE, false, 503, 21008 }
128128
};
@@ -283,7 +283,7 @@ public void faultInjectionServerErrorRuleTests_Partition() throws JsonProcessing
283283
cosmosDiagnostics,
284284
OperationType.Read,
285285
HttpConstants.StatusCodes.TOO_MANY_REQUESTS,
286-
HttpConstants.SubStatusCodes.UNKNOWN,
286+
HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE,
287287
feedRangeRuleId,
288288
true
289289
);
@@ -504,7 +504,7 @@ public void faultInjectionServerErrorRuleTests_HitLimit(
504504
cosmosDiagnostics,
505505
operationType,
506506
HttpConstants.StatusCodes.TOO_MANY_REQUESTS,
507-
HttpConstants.SubStatusCodes.UNKNOWN,
507+
HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE,
508508
hitLimitRuleId,
509509
true
510510
);

0 commit comments

Comments
 (0)