Skip to content

Commit 32018df

Browse files
piotrmadrypiotrmadry
authored
Merge pull request #23 from piotrmadry/add-instrumented-sharding
Add instrumented sharding
2 parents f58accf + 3f7e655 commit 32018df

File tree

11 files changed

+266
-68
lines changed

11 files changed

+266
-68
lines changed

‎plugin/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ apply plugin: "com.gradle.plugin-publish"
66
apply plugin: "org.gradle.kotlin.kotlin-dsl"
77

88
group = "firebase.test.lab"
9-
version = "1.1.6"
9+
version = "1.1.7"
1010

1111
gradlePlugin {
1212
plugins {

‎plugin/src/main/java/com/appunite/firebasetestlabplugin/FirebaseTestLabPlugin.kt‎

Lines changed: 105 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@ import com.android.build.gradle.api.BaseVariantOutput
66
import com.android.build.gradle.api.TestVariant
77
import com.appunite.firebasetestlabplugin.cloud.CloudTestResultDownloader
88
import com.appunite.firebasetestlabplugin.cloud.FirebaseTestLabProcessCreator
9+
import com.appunite.firebasetestlabplugin.cloud.ProcessData
910
import com.appunite.firebasetestlabplugin.cloud.TestType
1011
import com.appunite.firebasetestlabplugin.model.Device
1112
import com.appunite.firebasetestlabplugin.model.TestResults
13+
import com.appunite.firebasetestlabplugin.tasks.InstrumentationShardingTask
1214
import com.appunite.firebasetestlabplugin.utils.Constants
1315
import org.apache.tools.ant.taskdefs.condition.Os
1416
import org.gradle.api.GradleException
@@ -20,8 +22,9 @@ import org.gradle.kotlin.dsl.closureOf
2022
import org.gradle.kotlin.dsl.register
2123
import java.io.ByteArrayOutputStream
2224
import java.io.File
25+
import java.io.Serializable
2326

24-
internal class FirebaseTestLabPlugin : Plugin<Project> {
27+
class FirebaseTestLabPlugin : Plugin<Project> {
2528

2629
open class HiddenExec : Exec() {
2730
init {
@@ -66,7 +69,7 @@ internal class FirebaseTestLabPlugin : Plugin<Project> {
6669
}
6770
}
6871

69-
data class Sdk(val gcloud: File, val gsutil: File)
72+
data class Sdk(val gcloud: File, val gsutil: File): Serializable
7073

7174
private fun createDownloadSdkTask(project: Project, cloudSdkPath: String?): Sdk =
7275
if (cloudSdkPath != null) {
@@ -187,16 +190,9 @@ internal class FirebaseTestLabPlugin : Plugin<Project> {
187190
throw IllegalStateException("If you want to clear directory before run you need to setup cloudBucketName and cloudDirectoryName")
188191
}
189192

190-
val firebaseTestLabProcessCreator = FirebaseTestLabProcessCreator(
191-
sdk,
192-
cloudBucketName,
193-
cloudDirectoryName,
194-
project.logger
195-
)
196-
197193
(project.extensions.findByName(ANDROID) as AppExtension).apply {
198194
testVariants.toList().forEach { testVariant ->
199-
createGroupedTestLabTask(devices, testVariant, firebaseTestLabProcessCreator, ignoreFailures, downloader)
195+
createGroupedTestLabTask(devices, testVariant, ignoreFailures, downloader, sdk, cloudBucketName, cloudDirectoryName)
200196
}
201197
}
202198

@@ -206,14 +202,17 @@ internal class FirebaseTestLabPlugin : Plugin<Project> {
206202

207203
data class DeviceAppMap(val device: Device, val apk: BaseVariantOutput)
208204

209-
data class Test(val device: Device, val apk: BaseVariantOutput, val testApk: BaseVariantOutput)
210-
205+
data class Test(val device: Device, val apk: BaseVariantOutput, val testApk: BaseVariantOutput): Serializable
206+
211207
private fun createGroupedTestLabTask(
212-
devices: List<Device>,
213-
variant: TestVariant,
214-
firebaseTestLabProcessCreator: FirebaseTestLabProcessCreator,
215-
ignoreFailures: Boolean,
216-
downloader: CloudTestResultDownloader?) {
208+
devices: List<Device>,
209+
variant: TestVariant,
210+
ignoreFailures: Boolean,
211+
downloader: CloudTestResultDownloader?,
212+
sdk: Sdk,
213+
cloudBucketName: String?,
214+
cloudDirectoryName: String?
215+
) {
217216
val variantName = variant.testedVariant?.name?.capitalize() ?: ""
218217

219218
val cleanTask = "firebaseTestLabClean${variantName.capitalize()}"
@@ -264,19 +263,73 @@ internal class FirebaseTestLabPlugin : Plugin<Project> {
264263
dependsOn(taskSetup)
265264
dependsOn(arrayOf(test.apk.assemble))
266265
doLast {
267-
val result = firebaseTestLabProcessCreator.callFirebaseTestLab(test.device, test.apk.outputFile, TestType.Robo)
266+
val result = FirebaseTestLabProcessCreator.callFirebaseTestLab(ProcessData(
267+
sdk = sdk,
268+
gCloudBucketName = cloudBucketName,
269+
gCloudDirectory = cloudDirectoryName,
270+
device = test.device,
271+
apk = test.apk.outputFile,
272+
273+
testType = TestType.Robo
274+
))
268275
processResult(result, ignoreFailures)
269276
}
270277
})
271278
}
272-
273-
val instrumentationTasks = combineAll(appVersions, variant.outputs, {deviceAndMap, testApk -> Test(deviceAndMap.device, deviceAndMap.apk, testApk)})
274-
.map {
275-
test ->
276-
val devicePart = test.device.name.capitalize()
277-
val apkPart = dashToCamelCase(test.apk.name).capitalize()
278-
val testApkPart = test.testApk.let { if (it.filters.isEmpty()) "" else dashToCamelCase(it.name).capitalize() }
279-
val taskName = "$runTestsTaskInstrumentation$devicePart$apkPart$testApkPart"
279+
280+
val testResultFile = File(project.buildDir, "TestResults.txt")
281+
282+
val instrumentationTasks: List<Task> = combineAll(appVersions, variant.outputs)
283+
{ deviceAndMap, testApk -> Test(deviceAndMap.device, deviceAndMap.apk, testApk) }
284+
.map { test ->
285+
val devicePart = test.device.name.capitalize()
286+
val apkPart = dashToCamelCase(test.apk.name).capitalize()
287+
val testApkPart = test.testApk.let { if (it.filters.isEmpty()) "" else dashToCamelCase(it.name).capitalize() }
288+
val taskName = "$runTestsTaskInstrumentation$devicePart$apkPart$testApkPart"
289+
val numShards = test.device.numShards
290+
291+
if (numShards > 0) {
292+
project.tasks.create(taskName, InstrumentationShardingTask::class.java) {
293+
group = Constants.FIREBASE_TEST_LAB
294+
description = "Run Instrumentation test for ${test.device.name} device on $variantName/${test.apk.name} in Firebase Test Lab"
295+
this.processData = ProcessData(
296+
sdk = sdk,
297+
gCloudBucketName = cloudBucketName,
298+
gCloudDirectory = cloudDirectoryName,
299+
device = test.device,
300+
apk = test.apk.outputFile,
301+
testType = TestType.Instrumentation(test.testApk.outputFile)
302+
)
303+
this.stateFile = testResultFile
304+
305+
if (downloader != null) {
306+
mustRunAfter(cleanTask)
307+
}
308+
dependsOn(taskSetup)
309+
dependsOn(arrayOf(test.apk.assemble, test.testApk.assemble))
310+
311+
doFirst {
312+
testResultFile.writeText("")
313+
}
314+
315+
doLast {
316+
val testResults = testResultFile.readText()
317+
val resultCode: Int? = testResults.toIntOrNull()
318+
319+
logger.lifecycle("TESTS RESULTS: Every digit represents single shard.")
320+
logger.lifecycle("\"0\" means -> tests for particular shard passed.")
321+
logger.lifecycle("\"1\" means -> tests for particular shard failed.")
322+
323+
logger.lifecycle("RESULTS_CODE: $resultCode")
324+
logger.lifecycle("When result code is equal to 0 means that all tests for all shards passed, otherwise some of them failed.")
325+
326+
if (resultCode != null) {
327+
processResult(resultCode, ignoreFailures)
328+
}
329+
}
330+
}
331+
332+
} else {
280333
project.task(taskName, closureOf<Task> {
281334
inputs.files(test.testApk.outputFile, test.apk.outputFile)
282335
group = Constants.FIREBASE_TEST_LAB
@@ -287,13 +340,21 @@ internal class FirebaseTestLabPlugin : Plugin<Project> {
287340
dependsOn(taskSetup)
288341
dependsOn(arrayOf(test.apk.assemble, test.testApk.assemble))
289342
doLast {
290-
val result = firebaseTestLabProcessCreator.callFirebaseTestLab(test.device, test.apk.outputFile, TestType.Instrumentation(test.testApk.outputFile))
343+
val result = FirebaseTestLabProcessCreator.callFirebaseTestLab(ProcessData(
344+
sdk = sdk,
345+
gCloudBucketName = cloudBucketName,
346+
gCloudDirectory = cloudDirectoryName,
347+
device = test.device,
348+
apk = test.apk.outputFile,
349+
testType = TestType.Instrumentation(test.testApk.outputFile)
350+
))
291351
processResult(result, ignoreFailures)
292352
}
293353
})
294354
}
355+
}
295356

296-
val allInstrumentation = project.task(runTestsTaskInstrumentation, closureOf<Task> {
357+
val allInstrumentation: Task = project.task(runTestsTaskInstrumentation, closureOf<Task> {
297358
group = Constants.FIREBASE_TEST_LAB
298359
description = "Run all Instrumentation tests for $variantName in Firebase Test Lab"
299360
dependsOn(instrumentationTasks)
@@ -314,7 +375,7 @@ internal class FirebaseTestLabPlugin : Plugin<Project> {
314375
}
315376
})
316377

317-
val allRobo = project.task(runTestsTaskRobo, closureOf<Task> {
378+
val allRobo: Task = project.task(runTestsTaskRobo, closureOf<Task> {
318379
group = Constants.FIREBASE_TEST_LAB
319380
description = "Run all Robo tests for $variantName in Firebase Test Lab"
320381
dependsOn(roboTasks)
@@ -340,7 +401,7 @@ internal class FirebaseTestLabPlugin : Plugin<Project> {
340401
description = "Run all tests for $variantName in Firebase Test Lab"
341402
dependsOn(allRobo, allInstrumentation)
342403
})
343-
404+
344405
if (downloader != null) {
345406
project.task(downloadTask, closureOf<Task> {
346407
group = Constants.FIREBASE_TEST_LAB
@@ -355,7 +416,7 @@ internal class FirebaseTestLabPlugin : Plugin<Project> {
355416
})
356417
}
357418
}
358-
419+
359420
private fun processResult(result: TestResults, ignoreFailures: Boolean) {
360421
if (result.isSuccessful) {
361422
project.logger.lifecycle(result.message)
@@ -367,11 +428,22 @@ internal class FirebaseTestLabPlugin : Plugin<Project> {
367428
}
368429
}
369430
}
431+
432+
private fun processResult(resultCode: Int, ignoreFailures: Boolean) =
433+
if (resultCode == 0) {
434+
project.logger.lifecycle("SUCCESS: All tests passed.")
435+
} else {
436+
if (ignoreFailures) {
437+
println("FAILURE: Tests failed.")
438+
project.logger.error("FAILURE: Tests failed.")
439+
} else {
440+
throw GradleException("FAILURE: Tests failed.")
441+
}
442+
}
370443
}
371444

372-
373445
private fun <T1, T2, R> combineAll(l1: Collection<T1>, l2: Collection<T2>, func: (T1, T2) -> R): List<R> =
374446
l1.flatMap { t1 -> l2.map { t2 -> func(t1, t2)} }
375447

376-
private fun dashToCamelCase(dash: String): String =
448+
fun dashToCamelCase(dash: String): String =
377449
dash.split('-', '_').joinToString("") { it.capitalize() }

‎plugin/src/main/java/com/appunite/firebasetestlabplugin/cloud/FirebaseTestLabProcessCreator.kt‎

Lines changed: 47 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,25 @@ import com.appunite.firebasetestlabplugin.FirebaseTestLabPlugin
44
import com.appunite.firebasetestlabplugin.model.Device
55
import com.appunite.firebasetestlabplugin.model.TestResults
66
import com.appunite.firebasetestlabplugin.utils.joinArgs
7-
import org.gradle.api.logging.Logger
87
import java.io.File
8+
import java.io.Serializable
99

10-
sealed class TestType {
10+
sealed class TestType : Serializable {
1111
object Robo : TestType()
1212
data class Instrumentation(val testApk: File) : TestType()
1313
}
1414

15-
internal class FirebaseTestLabProcessCreator(
16-
private val sdk: FirebaseTestLabPlugin.Sdk,
17-
private val gCloudBucketName: String?,
18-
private val gCloudDirectory: String?,
19-
private val logger: Logger) {
15+
data class ProcessData(
16+
val sdk: FirebaseTestLabPlugin.Sdk,
17+
val gCloudBucketName: String?,
18+
val gCloudDirectory: String?,
19+
val device: Device,
20+
val apk: File,
21+
val testType: TestType,
22+
val shardIndex: Int = -1
23+
) : Serializable
24+
25+
object FirebaseTestLabProcessCreator {
2026

2127
private val resultMessageMap = mapOf(
2228
0 to "All test executions passed.",
@@ -29,40 +35,54 @@ internal class FirebaseTestLabProcessCreator(
2935
20 to "A test infrastructure error occurred."
3036
)
3137

32-
fun callFirebaseTestLab(device: Device, apk: File, testType: TestType): TestResults {
33-
val processBuilder = ProcessBuilder(
38+
fun callFirebaseTestLab(processData: ProcessData): TestResults {
39+
val process: Process = createProcess(processData).start()
40+
val resultCode: Int = process.let {
41+
it.errorStream.bufferedReader().forEachLine { errorInfo -> println(errorInfo) }
42+
it.inputStream.bufferedReader().forEachLine { info -> println(info) }
43+
process.waitFor()
44+
}
45+
return TestResults(
46+
isSuccessful = resultCode == 0,
47+
message = resultMessageMap.getOrElse(resultCode) { "Unknown error with code: $resultCode" }
48+
)
49+
}
50+
51+
private fun createProcess(processData: ProcessData): ProcessBuilder {
52+
val device: Device = processData.device
53+
return ProcessBuilder(
3454
sequenceOf(
35-
sdk.gcloud.absolutePath,
55+
processData.sdk.gcloud.absolutePath,
3656
"firebase", "test", "android", "run",
3757
"--format=json",
3858
"--device-ids=${device.deviceIds.joinArgs()}",
39-
"--app=$apk",
59+
"--app=${processData.apk}",
4060
"--locales=${device.locales.joinArgs()}",
4161
"--os-version-ids=${device.androidApiLevels.joinArgs()}",
4262
"--orientations=${device.screenOrientations.map { orientation -> orientation.gcloudName }.joinArgs()}")
43-
.plus(when (testType) {
63+
.plus(when (processData.testType) {
4464
TestType.Robo -> sequenceOf("--type=robo")
45-
is TestType.Instrumentation -> sequenceOf("--type=instrumentation", "--test=${testType.testApk}")
65+
is TestType.Instrumentation -> sequenceOf("--type=instrumentation", "--test=${processData.testType.testApk}")
4666
})
47-
.plus(gCloudBucketName?.let { sequenceOf("--results-bucket=$it") } ?: sequenceOf())
48-
.plus(gCloudDirectory?.let { sequenceOf("--results-dir=$it") } ?: sequenceOf())
67+
.plus(processData.gCloudBucketName?.let { sequenceOf("--results-bucket=$it") } ?: sequenceOf())
68+
.plus(processData.gCloudDirectory?.let { sequenceOf("--results-dir=$it") } ?: sequenceOf())
4969
.plus(if (device.isUseOrchestrator) sequenceOf("--use-orchestrator") else sequenceOf())
50-
.plus(if (device.environmentVariables.isNotEmpty()) sequenceOf("--environment-variables=${device.environmentVariables.joinToString(",")}") else sequenceOf())
70+
.plus(setupEnvironmentVariables(device, processData.shardIndex))
5171
.plus(if (device.testTargets.isNotEmpty()) sequenceOf("--test-targets=${device.testTargets.joinToString(",")}") else sequenceOf())
5272
.plus(device.customParamsForGCloudTool)
5373
.plus(device.testRunnerClass?.let { sequenceOf("--test-runner-class=$it") } ?: sequenceOf())
5474
.plus(if (device.timeout > 0) sequenceOf("--timeout=${device.timeout}s") else sequenceOf())
5575
.toList()
5676
)
57-
logger.debug(processBuilder.command().joinToString(separator = " ", transform = { "\"$it\"" }))
58-
val process = processBuilder.start()
59-
process.errorStream.bufferedReader().forEachLine { logger.lifecycle(it) }
60-
process.inputStream.bufferedReader().forEachLine { logger.lifecycle(it) }
61-
62-
val resultCode = process.waitFor()
63-
return TestResults(
64-
isSuccessful = resultCode == 0,
65-
message = resultMessageMap[resultCode] ?: ""
66-
)
6777
}
68-
}
78+
79+
private fun setupEnvironmentVariables(device: Device, shardIndex: Int): Sequence<String> =
80+
if (device.environmentVariables.isNotEmpty() || device.numShards > 0)
81+
sequenceOf(StringBuilder()
82+
.append("--environment-variables=")
83+
.append(if (device.environmentVariables.isNotEmpty()) device.environmentVariables.joinToString(",") else "")
84+
.append(if (device.environmentVariables.isNotEmpty() && device.numShards > 0) "," else "")
85+
.append(if (device.numShards > 0) "numShards=${device.numShards},shardIndex=$shardIndex" else "")
86+
.toString())
87+
else sequenceOf()
88+
}

‎plugin/src/main/java/com/appunite/firebasetestlabplugin/model/Device.kt‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package com.appunite.firebasetestlabplugin.model
22

3-
class Device(val name: String) {
3+
import java.io.Serializable
4+
5+
class Device(val name: String) : Serializable {
46
var locales: List<String> = listOf("en")
57
var screenOrientations: List<ScreenOrientation> = listOf(ScreenOrientation.PORTRAIT)
68
var androidApiLevels: List<Int> = listOf()
@@ -12,6 +14,8 @@ class Device(val name: String) {
1214
var filterAbiSplits = false
1315
var abiSplits: Set<String> = setOf()
1416
var isUseOrchestrator = false
17+
var numShards = 0
18+
1519
var environmentVariables: List<String> = listOf()
1620
var customParamsForGCloudTool: List<String> = listOf()
1721
var testTargets: List<String> = listOf()

‎plugin/src/main/java/com/appunite/firebasetestlabplugin/tasks/FirebaseTestLabProcess.kt‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package com.appunite.firebasetestlabplugin.tasks
2+
3+
import com.appunite.firebasetestlabplugin.cloud.FirebaseTestLabProcessCreator
4+
import com.appunite.firebasetestlabplugin.cloud.ProcessData
5+
import org.gradle.api.GradleException
6+
import java.io.File
7+
import javax.inject.Inject
8+
9+
class FirebaseTestLabProcess @Inject constructor(
10+
private val processData: ProcessData,
11+
private val stateFile: File
12+
) : Runnable {
13+
override fun run() {
14+
try {
15+
val testResults = FirebaseTestLabProcessCreator.callFirebaseTestLab(processData)
16+
stateFile.appendText(text = if (testResults.isSuccessful) "0" else "1")
17+
} catch (e: Exception){
18+
throw GradleException("There was a problem with processing ${e.message}")
19+
}
20+
}
21+
}

0 commit comments

Comments
 (0)