Skip to content

Commit f5f68e6

Browse files
piotradamczyk5piotradamczyk5
authored
feat: A (Flank#1042)
feat: Added options for default time and average time
1 parent ee35c6e commit f5f68e6

20 files changed

+323
-31
lines changed

‎README.md

+16
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,14 @@ flank:
169169
## Default: false
170170
# smart-flank-disable-upload: false
171171

172+
## Enable using average time from previous tests duration when using SmartShard and tests did not run before.
173+
## Default: false
174+
# use-average-test-time-for-new-tests: true
175+
176+
## Set default test time used for calculating shards.
177+
## Default: 120.0
178+
# default-test-time: 15
179+
172180
## Disables sharding. Useful for parameterized tests.
173181
# disable-sharding: false
174182

@@ -385,6 +393,14 @@ flank:
385393
## Default: false
386394
# smart-flank-disable-upload: false
387395

396+
## Enable using average time from previous tests duration when using SmartShard and tests did not run before.
397+
## Default: false
398+
# use-average-test-time-for-new-tests: true
399+
400+
## Set default test time used for calculating shards.
401+
## Default: 120.0
402+
# default-test-time: 15
403+
388404
## Disables sharding. Useful for parameterized tests.
389405
# disable-sharding: false
390406

‎test_runner/flank.ios.yml

+8
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,14 @@ flank:
101101
## Default: false
102102
# smart-flank-disable-upload: false
103103

104+
## Enable using average time from previous tests duration when using SmartShard and tests did not run before.
105+
## Default: false
106+
# use-average-test-time-for-new-tests: true
107+
108+
## Set default test time used for calculating shards.
109+
## Default: 120.0
110+
# default-test-time: 15
111+
104112
## Disables sharding. Useful for parameterized tests.
105113
# disable-sharding: false
106114

‎test_runner/flank.yml

+8
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,14 @@ flank:
161161
## Default: false
162162
# smart-flank-disable-upload: false
163163

164+
## Enable using average time from previous tests duration when using SmartShard and tests did not run before.
165+
## Default: false
166+
# use-average-test-time-for-new-tests: true
167+
168+
## Set default test time used for calculating shards.
169+
## Default: 120.0
170+
# default-test-time: 15
171+
164172
## Disables sharding. Useful for parameterized tests.
165173
# disable-sharding: false
166174

‎test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

+2
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ AndroidArgs
5858
num-test-runs: $repeatTests
5959
smart-flank-gcs-path: $smartFlankGcsPath
6060
smart-flank-disable-upload: $smartFlankDisableUpload
61+
default-test-time: $defaultTestTime
62+
use-average-test-time-for-new-tests: $useAverageTestTimeForNewTests
6163
files-to-download:${ArgsToString.listToString(filesToDownload)}
6264
test-targets-always-run:${ArgsToString.listToString(testTargetsAlwaysRun)}
6365
disable-sharding: $disableSharding

‎test_runner/src/main/kotlin/ftl/args/CommonArgs.kt

+3-1
Original file line numberDiff line numberDiff line change
@@ -34,5 +34,7 @@ data class CommonArgs(
3434
override val ignoreFailedTests: Boolean,
3535
override val keepFilePath: Boolean,
3636
override val outputStyle: OutputStyle,
37-
override val disableResultsUpload: Boolean
37+
override val disableResultsUpload: Boolean,
38+
override val defaultTestTime: Double,
39+
override val useAverageTestTimeForNewTests: Boolean,
3840
) : IArgs

‎test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt

+3-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@ fun CommonConfig.createCommonArgs(
4141
filesToDownload = flank.filesToDownload!!,
4242
disableSharding = flank.disableSharding!!,
4343
localResultDir = flank.localResultsDir!!,
44-
disableResultsUpload = flank.disableResultsUpload!!
44+
disableResultsUpload = flank.disableResultsUpload!!,
45+
defaultTestTime = flank.defaultTestTime!!,
46+
useAverageTestTimeForNewTests = flank.useAverageTestTimeForNewTests!!
4547
).apply {
4648
ArgsHelper.createJunitBucket(project, smartFlankGcsPath)
4749
}

‎test_runner/src/main/kotlin/ftl/args/IArgs.kt

+3
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ interface IArgs {
5959
val inVirtualRange: Boolean
6060
get() = maxTestShards in AVAILABLE_VIRTUAL_SHARD_COUNT_RANGE
6161

62+
val defaultTestTime: Double
63+
val useAverageTestTimeForNewTests: Boolean
64+
6265
fun useLocalResultDir() = localResultDir != defaultLocalResultsDir
6366

6467
companion object {

‎test_runner/src/main/kotlin/ftl/args/IosArgs.kt

+2
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ IosArgs
4343
num-test-runs: $repeatTests
4444
smart-flank-gcs-path: $smartFlankGcsPath
4545
smart-flank-disable-upload: $smartFlankDisableUpload
46+
default-test-time: $defaultTestTime
47+
use-average-test-time-for-new-tests: $useAverageTestTimeForNewTests
4648
test-targets-always-run:${ArgsToString.listToString(testTargetsAlwaysRun)}
4749
files-to-download:${ArgsToString.listToString(filesToDownload)}
4850
keep-file-path: $keepFilePath

‎test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt

+17
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import ftl.args.ArgsHelper
77
import ftl.args.yml.IYmlKeys
88
import ftl.config.Config
99
import ftl.config.FtlConstants
10+
import ftl.shard.DEFAULT_TEST_TIME_SEC
1011
import picocli.CommandLine
1112

1213
/** Flank specific parameters for both iOS and Android */
@@ -140,6 +141,20 @@ data class CommonFlankConfig @JsonIgnore constructor(
140141
@set:JsonProperty("disable-results-upload")
141142
var disableResultsUpload: Boolean? by data
142143

144+
@set:CommandLine.Option(
145+
names = ["--default-test-time"],
146+
description = ["Set default test time used for calculating shards."]
147+
)
148+
@set:JsonProperty("default-test-time")
149+
var defaultTestTime: Double? by data
150+
151+
@set:CommandLine.Option(
152+
names = ["--use-average-test-time-for-new-tests"],
153+
description = ["Enable using average time from previous tests duration when using SmartShard and tests did not run before."]
154+
)
155+
@set:JsonProperty("use-average-test-time-for-new-tests")
156+
var useAverageTestTimeForNewTests: Boolean? by data
157+
143158
constructor() : this(mutableMapOf<String, Any?>().withDefault { null })
144159

145160
companion object : IYmlKeys {
@@ -184,6 +199,8 @@ data class CommonFlankConfig @JsonIgnore constructor(
184199
keepFilePath = false
185200
outputStyle = null
186201
disableResultsUpload = false
202+
defaultTestTime = DEFAULT_TEST_TIME_SEC
203+
useAverageTestTimeForNewTests = false
187204
}
188205
}
189206
}

‎test_runner/src/main/kotlin/ftl/shard/Shard.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ fun createShardsByShardCount(
4545
val maxShards = maxShards(args.maxTestShards, forcedShardCount)
4646

4747
val previousMethodDurations = createTestMethodDurationMap(oldTestResult, args)
48-
val testCases = createTestCases(testsToRun, previousMethodDurations)
48+
val testCases = createTestCases(testsToRun, previousMethodDurations, args)
4949
.sortedByDescending(TestMethod::time) // We want to iterate over testcase going from slowest to fastest
5050

5151
val testCount = getNumberOfNotIgnoredTestCases(testCases)

‎test_runner/src/main/kotlin/ftl/shard/ShardCount.kt

+20-8
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,26 @@ private fun calculateShardCount(
2626
testsToRun: List<FlankTestMethod>,
2727
oldTestResult: JUnitTestResult,
2828
args: IArgs
29-
): Int = calculateShardCount(
30-
args = args,
31-
testsTotalTime = testTotalTime(testsToRun, createTestMethodDurationMap(oldTestResult, args)),
32-
testsToRunCount = testsToRun.size
33-
)
34-
35-
private fun testTotalTime(testsToRun: List<FlankTestMethod>, previousMethodDurations: Map<String, Double>): Double =
36-
testsToRun.sumByDouble { flankTestMethod -> getTestMethodTime(flankTestMethod, previousMethodDurations) }
29+
): Int {
30+
val previousMethodDurations = createTestMethodDurationMap(oldTestResult, args)
31+
return calculateShardCount(
32+
args = args,
33+
testsTotalTime = testTotalTime(testsToRun, previousMethodDurations, args.fallbackTestTime(previousMethodDurations)),
34+
testsToRunCount = testsToRun.size
35+
)
36+
}
37+
38+
private fun testTotalTime(
39+
testsToRun: List<FlankTestMethod>,
40+
previousMethodDurations: Map<String, Double>,
41+
defaultTestTime: Double
42+
) = testsToRun.sumByDouble { flankTestMethod ->
43+
getTestMethodTime(
44+
flankTestMethod,
45+
previousMethodDurations,
46+
defaultTestTime
47+
)
48+
}
3749

3850
private fun calculateShardCount(
3951
args: IArgs,

‎test_runner/src/main/kotlin/ftl/shard/TestCasesCreator.kt

+18-2
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,28 @@
11
package ftl.shard
22

3+
import ftl.args.IArgs
34
import ftl.util.FlankTestMethod
45

56
data class TestMethod(
67
val name: String,
78
val time: Double
89
)
910

10-
fun createTestCases(testsToRun: List<FlankTestMethod>, previousMethodDurations: Map<String, Double>): List<TestMethod> {
11-
return testsToRun.map { TestMethod(it.testName, getTestMethodTime(it, previousMethodDurations)) }
11+
fun createTestCases(
12+
testsToRun: List<FlankTestMethod>,
13+
previousMethodDurations: Map<String, Double>,
14+
args: IArgs,
15+
): List<TestMethod> {
16+
val defaultTestTime = args.fallbackTestTime(previousMethodDurations)
17+
return testsToRun
18+
.map {
19+
TestMethod(
20+
name = it.testName,
21+
time = getTestMethodTime(
22+
flankTestMethod = it,
23+
previousMethodDurations = previousMethodDurations,
24+
defaultTestTime = defaultTestTime
25+
)
26+
)
27+
}
1228
}

‎test_runner/src/main/kotlin/ftl/shard/TestMethodTime.kt

+19-11
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,26 @@
11
package ftl.shard
22

3-
import com.google.common.annotations.VisibleForTesting
3+
import ftl.args.IArgs
44
import ftl.util.FlankTestMethod
55

6-
// When a test does not have previous results to reference, fall back to this run time.
7-
@VisibleForTesting
8-
const val DEFAULT_TEST_TIME_SEC = 120.0
6+
fun getTestMethodTime(
7+
flankTestMethod: FlankTestMethod,
8+
previousMethodDurations: Map<String, Double>,
9+
defaultTestTime: Double
10+
) = if (flankTestMethod.ignored) IGNORE_TEST_TIME else previousMethodDurations.getOrElse(flankTestMethod.testName) {
11+
defaultTestTime
12+
}
13+
14+
fun IArgs.fallbackTestTime(
15+
previousMethodDurations: Map<String, Double>
16+
) = if (useAverageTestTimeForNewTests) previousMethodDurations.averageTestTime(defaultTestTime) else defaultTestTime
17+
18+
private fun Map<String, Double>.averageTestTime(defaultTestTime: Double) = values
19+
.filter { it > IGNORE_TEST_TIME }
20+
.average()
21+
.takeIf { !it.isNaN() }
22+
?: defaultTestTime
923

10-
@VisibleForTesting
1124
const val IGNORE_TEST_TIME = 0.0
1225

13-
fun getTestMethodTime(flankTestMethod: FlankTestMethod, previousMethodDurations: Map<String, Double>): Double {
14-
return if (flankTestMethod.ignored) IGNORE_TEST_TIME else previousMethodDurations.getOrDefault(
15-
flankTestMethod.testName,
16-
DEFAULT_TEST_TIME_SEC
17-
)
18-
}
26+
const val DEFAULT_TEST_TIME_SEC = 120.0

‎test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt

+61
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,8 @@ class AndroidArgsTest {
108108
max-test-shards: 7
109109
shard-time: 60
110110
num-test-runs: 8
111+
default-test-time: 15.0
112+
use-average-test-time-for-new-tests: true
111113
files-to-download:
112114
- /sdcard/screenshots
113115
- /sdcard/screenshots2
@@ -317,6 +319,8 @@ AndroidArgs
317319
num-test-runs: 8
318320
smart-flank-gcs-path:${' '}
319321
smart-flank-disable-upload: false
322+
default-test-time: 15.0
323+
use-average-test-time-for-new-tests: true
320324
files-to-download:
321325
- /sdcard/screenshots
322326
- /sdcard/screenshots2
@@ -382,6 +386,8 @@ AndroidArgs
382386
num-test-runs: 1
383387
smart-flank-gcs-path:
384388
smart-flank-disable-upload: false
389+
default-test-time: 120.0
390+
use-average-test-time-for-new-tests: false
385391
files-to-download:
386392
test-targets-always-run:
387393
disable-sharding: false
@@ -1676,6 +1682,61 @@ AndroidArgs
16761682
""".trimIndent()
16771683
AndroidArgs.load(yaml)
16781684
}
1685+
1686+
@Test
1687+
fun `should set defaultTestTime`() {
1688+
val yaml = """
1689+
gcloud:
1690+
app: $appApk
1691+
test: $testApk
1692+
flank:
1693+
max-test-shards: -1
1694+
default-test-time: 15
1695+
""".trimIndent()
1696+
val args = AndroidArgs.load(yaml)
1697+
assertEquals(args.defaultTestTime, 15.0, 0.01)
1698+
}
1699+
1700+
@Test
1701+
fun `should set defaultTestTime to default value if not specified`() {
1702+
val yaml = """
1703+
gcloud:
1704+
app: $appApk
1705+
test: $testApk
1706+
flank:
1707+
max-test-shards: -1
1708+
use-average-test-time-for-new-tests: true
1709+
""".trimIndent()
1710+
val args = AndroidArgs.load(yaml)
1711+
assertEquals(args.defaultTestTime, 120.0, 0.01)
1712+
}
1713+
1714+
@Test
1715+
fun `should useAverageTestTimeForNewTests set to true`() {
1716+
val yaml = """
1717+
gcloud:
1718+
app: $appApk
1719+
test: $testApk
1720+
flank:
1721+
max-test-shards: -1
1722+
use-average-test-time-for-new-tests: true
1723+
""".trimIndent()
1724+
val args = AndroidArgs.load(yaml)
1725+
assertTrue(args.useAverageTestTimeForNewTests)
1726+
}
1727+
1728+
@Test
1729+
fun `should useAverageTestTimeForNewTests set to false by defaul`() {
1730+
val yaml = """
1731+
gcloud:
1732+
app: $appApk
1733+
test: $testApk
1734+
flank:
1735+
max-test-shards: -1
1736+
""".trimIndent()
1737+
val args = AndroidArgs.load(yaml)
1738+
assertFalse(args.useAverageTestTimeForNewTests)
1739+
}
16791740
}
16801741

16811742
private fun AndroidArgs.Companion.load(yamlData: String, cli: AndroidRunCommand? = null): AndroidArgs =

0 commit comments

Comments
 (0)