Skip to content

Commit 5495bc1

Browse files
kushagraThaparkushagraThapar
authored
Separated flaky live tests to their own pipelines (Azure#37036)
* Separated flaky live tests to their own pipelines * Removed test directory from azure-cosmos project folder as the tests have been moved to azure-cosmos-test module * Removed profiles from azure-cosmos pom.xml since there are no tests being run from this module * Rearranged change feed split tests to start first * Experiment with max parallel * Experiment with max parallel
1 parent 4652e82 commit 5495bc1

File tree

82 files changed

+530
-1165
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

82 files changed

+530
-1165
lines changed

‎.vscode/cspell.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -686,7 +686,9 @@
686686
"Pdirect",
687687
"Pmulti",
688688
"Psplit",
689-
"Pquery"
689+
"Pquery",
690+
"Pcfp",
691+
"Pflaky"
690692
]
691693
},
692694
{

‎sdk/cosmos/azure-cosmos-benchmark/pom.xml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,27 @@ Licensed under the MIT License.
346346
</plugins>
347347
</build>
348348
</profile>
349+
<profile>
350+
<!-- integration tests, requires Cosmos DB endpoint -->
351+
<id>cfp-split</id>
352+
<properties>
353+
<test.groups>cfp-split</test.groups>
354+
</properties>
355+
<build>
356+
<plugins>
357+
<plugin>
358+
<groupId>org.apache.maven.plugins</groupId>
359+
<artifactId>maven-failsafe-plugin</artifactId>
360+
<version>3.1.0</version> <!-- {x-version-update;org.apache.maven.plugins:maven-failsafe-plugin;external_dependency} -->
361+
<configuration>
362+
<suiteXmlFiles>
363+
<suiteXmlFile>src/test/resources/cfp-split-testng.xml</suiteXmlFile>
364+
</suiteXmlFiles>
365+
</configuration>
366+
</plugin>
367+
</plugins>
368+
</build>
369+
</profile>
349370
<profile>
350371
<!-- integration tests, requires Cosmos DB endpoint -->
351372
<id>query</id>
@@ -430,6 +451,27 @@ Licensed under the MIT License.
430451
</plugins>
431452
</build>
432453
</profile>
454+
<profile>
455+
<!-- integration tests, requires Cosmos DB endpoint with multi master support -->
456+
<id>flaky-multi-master</id>
457+
<properties>
458+
<test.groups>flaky-multi-master</test.groups>
459+
</properties>
460+
<build>
461+
<plugins>
462+
<plugin>
463+
<groupId>org.apache.maven.plugins</groupId>
464+
<artifactId>maven-failsafe-plugin</artifactId>
465+
<version>3.1.0</version> <!-- {x-version-update;org.apache.maven.plugins:maven-failsafe-plugin;external_dependency} -->
466+
<configuration>
467+
<suiteXmlFiles>
468+
<suiteXmlFile>src/test/resources/flaky-multi-master-testng.xml</suiteXmlFile>
469+
</suiteXmlFiles>
470+
</configuration>
471+
</plugin>
472+
</plugins>
473+
</build>
474+
</profile>
433475
<profile>
434476
<!-- integration tests, requires Cosmos DB endpoint -->
435477
<id>examples</id>

‎sdk/cosmos/azure-cosmos-benchmark/src/test/java/com/azure/cosmos/benchmark/WorkflowTest.java‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public class WorkflowTest {
3030
private Database database;
3131
private DocumentCollection collection;
3232

33-
@Test(groups = "simple", timeOut = TIMEOUT)
33+
@Test(groups = "fast", timeOut = TIMEOUT)
3434
public void readMyWritesCLI() throws Exception {
3535
String cmdFormat = "-serviceEndpoint %s -masterKey %s" +
3636
" -databaseId %s -collectionId %s" +
@@ -45,7 +45,7 @@ public void readMyWritesCLI() throws Exception {
4545
Main.main(StringUtils.split(cmd));
4646
}
4747

48-
@Test(dataProvider = "collectionLinkTypeArgProvider", groups = "simple", timeOut = TIMEOUT)
48+
@Test(dataProvider = "collectionLinkTypeArgProvider", groups = "fast", timeOut = TIMEOUT)
4949
public void readMyWrites(boolean useNameLink) throws Exception {
5050
int numberOfOperations = 123;
5151
String cmdFormat = "-serviceEndpoint %s -masterKey %s" +
@@ -86,7 +86,7 @@ protected void onSuccess() {
8686
assertThat(success).hasValue(numberOfOperations);
8787
}
8888

89-
@Test(groups = "simple", timeOut = TIMEOUT)
89+
@Test(groups = "fast", timeOut = TIMEOUT)
9090
public void writeLatencyCLI() throws Exception {
9191
String cmdFormat = "-serviceEndpoint %s -masterKey %s" +
9292
" -databaseId %s -collectionId %s" +
@@ -101,7 +101,7 @@ public void writeLatencyCLI() throws Exception {
101101
Main.main(StringUtils.split(cmd));
102102
}
103103

104-
@Test(dataProvider = "collectionLinkTypeArgProvider", groups = "simple", timeOut = TIMEOUT)
104+
@Test(dataProvider = "collectionLinkTypeArgProvider", groups = "fast", timeOut = TIMEOUT)
105105
public void writeLatency(boolean useNameLink) throws Exception {
106106
int numberOfOperations = 123;
107107
String cmdFormat = "-serviceEndpoint %s -masterKey %s" +
@@ -142,7 +142,7 @@ protected void onSuccess() {
142142
assertThat(success).hasValue(numberOfOperations);
143143
}
144144

145-
@Test(dataProvider = "collectionLinkTypeArgProvider", groups = "simple", timeOut = TIMEOUT)
145+
@Test(dataProvider = "collectionLinkTypeArgProvider", groups = "fast", timeOut = TIMEOUT)
146146
public void writeThroughput(boolean useNameLink) throws Exception {
147147
int numberOfOperations = 123;
148148
String cmdFormat = "-serviceEndpoint %s -masterKey %s" +
@@ -183,7 +183,7 @@ protected void onSuccess() {
183183
assertThat(success).hasValue(numberOfOperations);
184184
}
185185

186-
@Test(dataProvider = "collectionLinkTypeArgProvider", groups = "simple", timeOut = TIMEOUT)
186+
@Test(dataProvider = "collectionLinkTypeArgProvider", groups = "fast", timeOut = TIMEOUT)
187187
public void readLatency(boolean useNameLink) throws Exception {
188188
int numberOfOperations = 123;
189189
String cmdFormat = "-serviceEndpoint %s -masterKey %s" +
@@ -224,7 +224,7 @@ protected void onSuccess() {
224224
assertThat(success).hasValue(numberOfOperations);
225225
}
226226

227-
@Test(dataProvider = "collectionLinkTypeArgProvider", groups = "simple", timeOut = TIMEOUT)
227+
@Test(dataProvider = "collectionLinkTypeArgProvider", groups = "fast", timeOut = TIMEOUT)
228228
public void readThroughput(boolean useNameLink) throws Exception {
229229
int numberOfOperations = 123;
230230
String cmdFormat = "-serviceEndpoint %s -masterKey %s" +
@@ -265,7 +265,7 @@ protected void onSuccess() {
265265
assertThat(success).hasValue(numberOfOperations);
266266
}
267267

268-
@BeforeClass(groups = "simple", timeOut = TIMEOUT)
268+
@BeforeClass(groups = "fast", timeOut = TIMEOUT)
269269
public void before_WorkflowTest() {
270270
RequestOptions options = new RequestOptions();
271271
options.setOfferThroughput(10000);
@@ -286,7 +286,7 @@ public Object[][] collectionLinkTypeArgProvider() {
286286
};
287287
}
288288

289-
@AfterClass(groups = "simple", timeOut = TIMEOUT)
289+
@AfterClass(groups = "fast", timeOut = TIMEOUT)
290290
public void afterClass() {
291291
AsyncDocumentClient housekeepingClient = Utils.housekeepingClient();
292292
Utils.safeCleanDatabases(housekeepingClient);

sdk/cosmos/azure-cosmos/src/test/resources/direct-testng.xml renamed to sdk/cosmos/azure-cosmos-benchmark/src/test/resources/cfp-split-testng.xml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,11 @@
2121
~ SOFTWARE.
2222
-->
2323
<!DOCTYPE suite SYSTEM "https://testng.org/testng-1.0.dtd">
24-
<suite name="direct">
25-
<test name="direct" group-by-instances="true">
24+
<suite name="cfp-split">
25+
<test name="cfp-split" group-by-instances="true">
2626
<groups>
2727
<run>
28-
<include name="direct"/>
28+
<include name="cfp-split"/>
2929
</run>
3030
</groups>
3131
<packages>

‎sdk/cosmos/azure-cosmos-benchmark/src/test/resources/fast-testng.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
<test name="fast" group-by-instances="true">
2626
<groups>
2727
<run>
28-
<include name="simple"/>
28+
<include name="fast"/>
2929
</run>
3030
</groups>
3131
<packages>

sdk/cosmos/azure-cosmos/src/test/resources/emulator-testng.xml renamed to sdk/cosmos/azure-cosmos-benchmark/src/test/resources/flaky-multi-master-testng.xml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,11 @@
2121
~ SOFTWARE.
2222
-->
2323
<!DOCTYPE suite SYSTEM "https://testng.org/testng-1.0.dtd">
24-
<suite name="emulator">
25-
<test name="emulator" group-by-instances="true">
24+
<suite name="flaky-multi-master">
25+
<test name="flaky-multi-master" group-by-instances="true">
2626
<groups>
2727
<run>
28-
<include name="emulator"/>
28+
<include name="flaky-multi-master"/>
2929
</run>
3030
</groups>
3131
<packages>

‎sdk/cosmos/azure-cosmos-dotnet-benchmark/src/test/java/BasicOperationTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public class BasicOperationTest {
3434
private CosmosContainer createdTestContainer;
3535
private CosmosContainer createdResultsContainer;
3636

37-
@BeforeClass(groups = {"simple"}, timeOut = TIMEOUT)
37+
@BeforeClass(groups = {"fast"}, timeOut = TIMEOUT)
3838
public void before_BasicOperationTest() {
3939
assertThat(this.client).isNull();
4040
CosmosClientBuilder clientBuilder = new CosmosClientBuilder()
@@ -60,22 +60,22 @@ public void before_BasicOperationTest() {
6060
this.createdResultsContainer = this.createdDatabase.getContainer("results" + suffix);
6161
}
6262

63-
@AfterClass(groups = {"simple"}, timeOut = TIMEOUT, alwaysRun = true)
63+
@AfterClass(groups = {"fast"}, timeOut = TIMEOUT, alwaysRun = true)
6464
public void afterClass() {
6565
assertThat(this.client).isNotNull();
6666
assertThat(this.createdDatabase).isNotNull();
6767
this.createdDatabase.delete();
6868
this.client.close();
6969
}
7070

71-
@Test(groups = { "simple" }, timeOut = TIMEOUT)
71+
@Test(groups = { "fast" }, timeOut = TIMEOUT)
7272
public void Run_InsertWithoutExplicitPKAndId() throws Exception {
7373
this.runTest(
7474
"InsertWithoutExplicitPKAndId",
7575
2, 2000);
7676
}
7777

78-
@Test(groups = { "simple" }, timeOut = TIMEOUT)
78+
@Test(groups = { "fast" }, timeOut = TIMEOUT)
7979
public void Run_Insert() throws Exception {
8080
this.runTest(
8181
"Insert",

‎sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -215,7 +215,7 @@ public CosmosAsyncDatabase getDatabase(String id) {
215215
}
216216
}
217217

218-
@BeforeSuite(groups = {"simple", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SETUP_TIMEOUT)
218+
@BeforeSuite(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SETUP_TIMEOUT)
219219
public static void beforeSuite() {
220220

221221
logger.info("beforeSuite Started");
@@ -249,7 +249,7 @@ public static void beforeSuite() {
249249
}
250250
}
251251

252-
@AfterSuite(groups = {"simple", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SHUTDOWN_TIMEOUT)
252+
@AfterSuite(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SHUTDOWN_TIMEOUT)
253253
public static void afterSuite() {
254254

255255
logger.info("afterSuite Started");

‎sdk/cosmos/azure-cosmos-encryption/src/test/resources/fast-testng.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
<test name="fast" group-by-instances="true">
2626
<groups>
2727
<run>
28-
<include name="simple"/>
28+
<include name="fast"/>
2929
</run>
3030
</groups>
3131
<packages>

‎sdk/cosmos/azure-cosmos-tests/pom.xml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,27 @@ Licensed under the MIT License.
333333
</plugins>
334334
</build>
335335
</profile>
336+
<profile>
337+
<!-- integration tests, requires Cosmos DB endpoint -->
338+
<id>cfp-split</id>
339+
<properties>
340+
<test.groups>cfp-split</test.groups>
341+
</properties>
342+
<build>
343+
<plugins>
344+
<plugin>
345+
<groupId>org.apache.maven.plugins</groupId>
346+
<artifactId>maven-failsafe-plugin</artifactId>
347+
<version>3.1.0</version> <!-- {x-version-update;org.apache.maven.plugins:maven-failsafe-plugin;external_dependency} -->
348+
<configuration>
349+
<suiteXmlFiles>
350+
<suiteXmlFile>src/test/resources/cfp-split-testng.xml</suiteXmlFile>
351+
</suiteXmlFiles>
352+
</configuration>
353+
</plugin>
354+
</plugins>
355+
</build>
356+
</profile>
336357
<profile>
337358
<!-- integration tests, requires Cosmos DB endpoint -->
338359
<id>query</id>
@@ -417,6 +438,27 @@ Licensed under the MIT License.
417438
</plugins>
418439
</build>
419440
</profile>
441+
<profile>
442+
<!-- integration tests, requires Cosmos DB endpoint with multi master support -->
443+
<id>flaky-multi-master</id>
444+
<properties>
445+
<test.groups>flaky-multi-master</test.groups>
446+
</properties>
447+
<build>
448+
<plugins>
449+
<plugin>
450+
<groupId>org.apache.maven.plugins</groupId>
451+
<artifactId>maven-failsafe-plugin</artifactId>
452+
<version>3.1.0</version> <!-- {x-version-update;org.apache.maven.plugins:maven-failsafe-plugin;external_dependency} -->
453+
<configuration>
454+
<suiteXmlFiles>
455+
<suiteXmlFile>src/test/resources/flaky-multi-master-testng.xml</suiteXmlFile>
456+
</suiteXmlFiles>
457+
</configuration>
458+
</plugin>
459+
</plugins>
460+
</build>
461+
</profile>
420462
<profile>
421463
<!-- integration tests, requires Cosmos DB endpoint with multi region support -->
422464
<id>multi-region</id>

0 commit comments

Comments
 (0)