Skip to content

Commit a2340aa

Browse files
jan-goraljan-goral
authored
feat: Add test axis value column to summary table (Flank#1040)
* feat(output): add test axis value column to summary table docs(output): update summary table * Display full device name in outcome table * Update summary_output.md * Fix namings
1 parent 8fdc73b commit a2340aa

15 files changed

+144
-85
lines changed

‎docs/feature/summary_output.md

+6-5
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
# Formatted summary output
22
```
3-
┌─────────┬──────────────────────┬─────────────────────┐
4-
│ OUTCOME │ MATRIX ID │ TEST DETAILS │
5-
├─────────┼──────────────────────┼─────────────────────┤
6-
│ success │ matrix-1z85qtvdnvb0l │ 4 test cases passed │
7-
└─────────┴──────────────────────┴─────────────────────┘
3+
┌─────────┬──────────────────────┬──────────────────────────┬─────────────────────────────────────────┐
4+
│ OUTCOME │ MATRIX ID │ TEST AXIS VALUE │ TEST DETAILS │
5+
├─────────┼──────────────────────┼──────────────────────────┼─────────────────────────────────────────┤
6+
│ success │ matrix-35czp85w4h3a7 │ greatqlte-26-en-portrait │ 20 test cases passed │
7+
│ failure │ matrix-35czp85w4h3a7 │ Nexus6P-26-en-portrait │ 1 test cases failed, 16 passed, 3 flaky │
8+
└─────────┴──────────────────────┴──────────────────────────┴─────────────────────────────────────────┘
89
```
910

1011

‎test_runner/src/main/kotlin/ftl/json/MatrixMap.kt

+5-3
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,9 @@ class MatrixMap(
3939
*/
4040
fun validateMatrices(shouldIgnore: Boolean = false) {
4141
map.values.run {
42-
firstOrNull { it.canceledByUser() }?.let { throw MatrixCanceledError(it.outcomeDetails.orEmpty()) }
43-
firstOrNull { it.infrastructureFail() }?.let { throw InfrastructureError(it.outcomeDetails.orEmpty()) }
44-
firstOrNull { it.incompatibleFail() }?.let { throw IncompatibleTestDimensionError(it.outcomeDetails.orEmpty()) }
42+
firstOrNull { it.canceledByUser() }?.run { throw MatrixCanceledError(outcomeDetails) }
43+
firstOrNull { it.infrastructureFail() }?.run { throw InfrastructureError(outcomeDetails) }
44+
firstOrNull { it.incompatibleFail() }?.run { throw IncompatibleTestDimensionError(outcomeDetails) }
4545
firstOrNull { it.state != MatrixState.FINISHED }?.let { throw FTLError(it) }
4646
filter { it.isFailed() }.let {
4747
if (it.isNotEmpty()) throw FailedMatrixError(
@@ -53,6 +53,8 @@ class MatrixMap(
5353
}
5454
}
5555

56+
private val SavedMatrix.outcomeDetails get() = testAxises.firstOrNull()?.details.orEmpty()
57+
5658
fun Iterable<TestMatrix>.update(matrixMap: MatrixMap) = forEach { matrix ->
5759
matrixMap.map[matrix.testMatrixId]?.updateWithMatrix(matrix)?.let {
5860
matrixMap.map[matrix.testMatrixId] = it

‎test_runner/src/main/kotlin/ftl/json/OutcomeDetailsFormatter.kt

+3-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@ import ftl.util.StepOutcome.skipped
1313
import ftl.util.StepOutcome.success
1414
import ftl.util.StepOutcome.unset
1515

16-
internal fun Outcome.getDetails(testSuiteOverviewData: TestSuiteOverviewData?): String = when (summary) {
16+
internal fun Outcome?.getDetails(
17+
testSuiteOverviewData: TestSuiteOverviewData?
18+
): String = when (this?.summary) {
1719
success, flaky -> testSuiteOverviewData
1820
?.getSuccessOutcomeDetails(successDetail?.otherNativeCrash ?: false)
1921
?: "Unknown outcome"

‎test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt

+16-13
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import ftl.reports.outcome.createMatrixOutcomeSummary
77
import ftl.reports.outcome.fetchTestOutcomeContext
88
import ftl.util.MatrixState.FINISHED
99
import ftl.util.MatrixState.INVALID
10+
import ftl.util.StepOutcome
1011
import ftl.util.StepOutcome.failure
1112
import ftl.util.StepOutcome.inconclusive
1213
import ftl.util.StepOutcome.skipped
@@ -26,21 +27,24 @@ data class SavedMatrix(
2627
val downloaded: Boolean = false,
2728
val billableVirtualMinutes: Long = 0,
2829
val billablePhysicalMinutes: Long = 0,
29-
val outcome: String = "",
30-
val outcomeDetails: String = "",
3130
val clientDetails: Map<String, String>? = null,
3231
val gcsPathWithoutRootBucket: String = "",
3332
val gcsRootBucket: String = "",
3433
val webLinkWithoutExecutionDetails: String? = "",
35-
)
34+
val testAxises: List<TestOutcome> = emptyList()
35+
) {
36+
val outcome = testAxises.maxByOrNull { StepOutcome.order.indexOf(it.outcome) }?.outcome.orEmpty()
37+
}
3638

3739
fun createSavedMatrix(testMatrix: TestMatrix) = SavedMatrix().updateWithMatrix(testMatrix)
3840

39-
fun SavedMatrix.canceledByUser() = outcomeDetails == ABORTED_BY_USER_MESSAGE
41+
fun SavedMatrix.canceledByUser() = testAxises.any { it.details == ABORTED_BY_USER_MESSAGE }
42+
43+
fun SavedMatrix.infrastructureFail() = testAxises.any { it.details == INFRASTRUCTURE_FAILURE_MESSAGE }
4044

41-
fun SavedMatrix.infrastructureFail() = outcomeDetails == INFRASTRUCTURE_FAILURE_MESSAGE
45+
fun SavedMatrix.incompatibleFail() = testAxises.map { it.details }.intersect(incompatibleFails).isNotEmpty()
4246

43-
fun SavedMatrix.incompatibleFail() = outcomeDetails in arrayOf(
47+
private val incompatibleFails = setOf(
4448
INCOMPATIBLE_APP_VERSION_MESSAGE,
4549
INCOMPATIBLE_ARCHITECTURE_MESSAGE,
4650
INCOMPATIBLE_DEVICE_MESSAGE
@@ -70,11 +74,11 @@ private fun SavedMatrix.updatedSavedMatrix(
7074
): SavedMatrix = when (newMatrix.state) {
7175
state -> this
7276

73-
FINISHED -> newMatrix.fetchTestOutcomeContext().createMatrixOutcomeSummary().let { (billableMinutes, outcome) ->
74-
updateProperties(newMatrix).updateOutcome(outcome).updateBillableMinutes(billableMinutes)
77+
FINISHED -> newMatrix.fetchTestOutcomeContext().createMatrixOutcomeSummary().let { (billableMinutes, outcomes) ->
78+
updateProperties(newMatrix).updateOutcome(outcomes).updateBillableMinutes(billableMinutes)
7579
}
7680

77-
INVALID -> updateProperties(newMatrix).updateOutcome(invalidTestOutcome())
81+
INVALID -> updateProperties(newMatrix).updateOutcome(listOf(invalidTestOutcome()))
7882

7983
else -> updateProperties(newMatrix)
8084
}
@@ -96,12 +100,11 @@ private fun SavedMatrix.updateBillableMinutes(billableMinutes: BillableMinutes)
96100
billableVirtualMinutes = billableMinutes.virtual,
97101
)
98102

99-
private fun SavedMatrix.updateOutcome(testOutcome: TestOutcome) = copy(
100-
outcome = testOutcome.outcome,
101-
outcomeDetails = testOutcome.testDetails
103+
private fun SavedMatrix.updateOutcome(outcome: List<TestOutcome>) = copy(
104+
testAxises = outcome
102105
)
103106

104107
private fun invalidTestOutcome() = TestOutcome(
105108
outcome = "---",
106-
testDetails = "Matrix is invalid"
109+
details = "Matrix is invalid"
107110
)

‎test_runner/src/main/kotlin/ftl/json/SavedMatrixTableUtil.kt

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package ftl.json
2+
3+
import ftl.reports.outcome.TestOutcome
4+
import ftl.util.StepOutcome.failure
5+
import ftl.util.StepOutcome.flaky
6+
import ftl.util.StepOutcome.success
7+
import ftl.util.SystemOutColor
8+
import ftl.util.TableColumn
9+
import ftl.util.buildTable
10+
11+
fun SavedMatrix.asPrintableTable(): String = listOf(this).asPrintableTable()
12+
13+
fun List<SavedMatrix>.asPrintableTable(): String = buildTable(
14+
TableColumn(
15+
header = OUTCOME_COLUMN_HEADER,
16+
data = flatMapTestAxis { outcome },
17+
dataColor = flatMapTestAxis { outcomeColor }
18+
),
19+
TableColumn(
20+
header = MATRIX_ID_COLUMN_HEADER,
21+
data = flatMapTestAxis { matrix -> matrix.matrixId }
22+
),
23+
TableColumn(
24+
header = TEST_AXIS_VALUE_HEADER,
25+
data = flatMapTestAxis { device }
26+
),
27+
TableColumn(
28+
header = OUTCOME_DETAILS_COLUMN_HEADER,
29+
data = flatMapTestAxis { details }
30+
)
31+
)
32+
33+
private fun <T> List<SavedMatrix>.flatMapTestAxis(transform: TestOutcome.(SavedMatrix) -> T) =
34+
flatMap { matrix -> matrix.testAxises.map { axis -> axis.transform(matrix) } }
35+
36+
private val TestOutcome.outcomeColor
37+
get() = when (outcome) {
38+
failure -> SystemOutColor.RED
39+
success -> SystemOutColor.GREEN
40+
flaky -> SystemOutColor.BLUE
41+
else -> SystemOutColor.DEFAULT
42+
}
43+
44+
private const val OUTCOME_COLUMN_HEADER = "OUTCOME"
45+
private const val MATRIX_ID_COLUMN_HEADER = "MATRIX ID"
46+
private const val TEST_AXIS_VALUE_HEADER = "TEST AXIS VALUE"
47+
private const val OUTCOME_DETAILS_COLUMN_HEADER = "TEST DETAILS"

‎test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import ftl.json.SavedMatrix
88
import ftl.json.isFailed
99
import ftl.reports.util.IReport
1010
import ftl.reports.xml.model.JUnitTestResult
11-
import ftl.util.asPrintableTable
11+
import ftl.json.asPrintableTable
1212
import ftl.util.println
1313
import java.io.StringWriter
1414
import java.text.DecimalFormat

‎test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestSuite.kt

+2-7
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
package ftl.reports.api
22

3-
import com.google.api.services.toolresults.model.Step
43
import ftl.reports.api.data.TestExecutionData
54
import ftl.reports.api.data.TestSuiteOverviewData
5+
import ftl.reports.outcome.axisValue
66
import ftl.reports.xml.model.JUnitTestSuite
77

88
internal fun List<TestExecutionData>.createJUnitTestSuites() = mapNotNull { data: TestExecutionData ->
@@ -18,7 +18,7 @@ private fun createJUnitTestSuite(
1818
data: TestExecutionData,
1919
overview: TestSuiteOverviewData
2020
) = JUnitTestSuite(
21-
name = data.step.testSuiteName(),
21+
name = data.step.axisValue(),
2222
timestamp = data.timestamp.asUnixTimestamp().formatUtcDate(),
2323
tests = overview.total.toString(),
2424
failures = overview.failures.toString(),
@@ -32,8 +32,3 @@ private fun createJUnitTestSuite(
3232
).toMutableList(),
3333
time = overview.elapsedTime.format()
3434
)
35-
36-
private fun Step.testSuiteName(): String {
37-
val map = dimensionValue.map { it.key to it.value }.toMap()
38-
return listOf(map["Model"], map["Version"], map["Locale"], map["Orientation"]).joinToString("-")
39-
}

‎test_runner/src/main/kotlin/ftl/reports/outcome/BillableMinutes.kt

+1-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package ftl.reports.outcome
22

33
import com.google.api.services.toolresults.model.Step
4-
import com.google.api.services.toolresults.model.StepDimensionValueEntry
54
import ftl.android.AndroidCatalog
65
import ftl.util.billableMinutes
76
import kotlin.math.min
@@ -25,7 +24,7 @@ fun List<Step>.calculateAndroidBillableMinutes(
2524
private fun List<Step>.groupByDeviceType(projectId: String) =
2625
groupBy {
2726
AndroidCatalog.isVirtualDevice(
28-
it.deviceModel(),
27+
it.axisValue(),
2928
projectId
3029
)
3130
}
@@ -41,6 +40,3 @@ private fun Step.getBillableSeconds(default: Long) =
4140
testExecutionStep?.testTiming?.testProcessDuration?.seconds?.let {
4241
min(it, default)
4342
}
44-
45-
operator fun List<StepDimensionValueEntry>?.get(key: String) =
46-
this?.firstOrNull { it.key == key }

‎test_runner/src/main/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummary.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package ftl.reports.outcome
22

33
import com.google.api.services.toolresults.model.Environment
44

5-
fun TestOutcomeContext.createMatrixOutcomeSummary(): Pair<BillableMinutes, TestOutcome> =
5+
fun TestOutcomeContext.createMatrixOutcomeSummary(): Pair<BillableMinutes, List<TestOutcome>> =
66
steps.calculateAndroidBillableMinutes(projectId, testTimeout) to
77
if (environments.hasOutcome())
88
environments.createMatrixOutcomeSummaryUsingEnvironments()

‎test_runner/src/main/kotlin/ftl/reports/outcome/CreateTestSuiteOverviewData.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ internal fun Environment.createTestSuiteOverviewData(): TestSuiteOverviewData =
2020
internal fun List<Step>.createTestSuiteOverviewData(): TestSuiteOverviewData = this
2121
.also { require(isNotEmpty()) }
2222
.filter(Step::isPrimaryStep)
23-
.groupBy(Step::deviceModel)
23+
.groupBy(Step::axisValue)
2424
.values
2525
.map { it.mapToTestSuiteOverviews().foldTestSuiteOverviewData() }
2626
.fold(TestSuiteOverviewData()) { acc, data -> acc + data } // Fixme https://github.com/Flank/flank/issues/983

‎test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcome.kt

+21-14
Original file line numberDiff line numberDiff line change
@@ -7,30 +7,37 @@ import ftl.json.getDetails
77
import ftl.util.StepOutcome
88

99
data class TestOutcome(
10-
val outcome: String,
11-
val testDetails: String
10+
val device: String = "",
11+
val outcome: String = "",
12+
val details: String = "",
1213
)
1314

14-
fun List<Environment>.createMatrixOutcomeSummaryUsingEnvironments(
15-
outcome: Outcome? = getOutcomeFromEnvironments(),
16-
testDetails: String? = outcome?.getDetails(map { it.createTestSuiteOverviewData() }.foldTestSuiteOverviewData())
15+
fun List<Environment>.createMatrixOutcomeSummaryUsingEnvironments(): List<TestOutcome> =
16+
map(Environment::getTestOutcome)
17+
18+
private fun Environment.getTestOutcome(
19+
outcome: Outcome? = environmentResult?.outcome
1720
) = TestOutcome(
18-
outcome = outcome?.summary ?: "Unknown",
19-
testDetails = testDetails ?: "Unknown outcome"
21+
device = axisValue(),
22+
outcome = outcome?.summary ?: UNKNOWN_OUTCOME,
23+
details = outcome.getDetails(createTestSuiteOverviewData()),
2024
)
2125

22-
private fun List<Environment>.getOutcomeFromEnvironments(): Outcome? = maxByOrNull {
23-
StepOutcome.order.indexOf(it.environmentResult?.outcome?.summary)
24-
}?.environmentResult?.outcome
26+
fun List<Step>.createMatrixOutcomeSummaryUsingSteps() = groupBy(Step::axisValue).map { (device, steps) ->
27+
steps.getTestOutcome(device)
28+
}
2529

26-
fun List<Step>.createMatrixOutcomeSummaryUsingSteps(
30+
private fun List<Step>.getTestOutcome(
31+
deviceModel: String,
2732
outcome: Outcome? = getOutcomeFromSteps(),
28-
testDetails: String? = outcome?.getDetails(createTestSuiteOverviewData())
2933
) = TestOutcome(
30-
outcome = outcome?.summary ?: "Unknown",
31-
testDetails = testDetails ?: "Unknown outcome"
34+
device = deviceModel,
35+
outcome = outcome?.summary ?: UNKNOWN_OUTCOME,
36+
details = outcome.getDetails(createTestSuiteOverviewData())
3237
)
3338

3439
private fun List<Step>.getOutcomeFromSteps(): Outcome? = maxByOrNull {
3540
StepOutcome.order.indexOf(it.outcome?.summary)
3641
}?.outcome
42+
43+
private const val UNKNOWN_OUTCOME = "Unknown"

‎test_runner/src/main/kotlin/ftl/reports/outcome/Util.kt

+24-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,29 @@
11
package ftl.reports.outcome
22

3+
import com.google.api.client.json.GenericJson
4+
import com.google.api.services.toolresults.model.Environment
35
import com.google.api.services.toolresults.model.Step
46
import ftl.environment.orUnknown
7+
import ftl.util.mutableMapProperty
58

6-
internal fun Step.deviceModel() = dimensionValue["Model"]
7-
?.value.orUnknown()
9+
internal fun Step.axisValue() = dimensionValue.axisValue()
10+
11+
internal fun Environment.axisValue() = dimensionValue.axisValue()
12+
13+
private fun List<GenericJson>?.axisValue() = this
14+
?.toDimensionMap()
15+
?.getValues(dimensionKeys)
16+
?.joinToString("-")
17+
.orUnknown()
18+
19+
private fun List<GenericJson>.toDimensionMap(): Map<String?, String?> = associate { it.key to it.value }
20+
21+
private fun Map<String?, String?>.getValues(keys: Iterable<String>) = keys.mapNotNull { key -> get(key) }
22+
23+
private val GenericJson.key: String? by mutableMapProperty { null }
24+
25+
private val GenericJson.value: String? by mutableMapProperty { null }
26+
27+
private val dimensionKeys = DimensionValue.values().map(DimensionValue::name)
28+
29+
private enum class DimensionValue { Model, Version, Locale, Orientation }

‎test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt

-25
This file was deleted.

‎test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt

+3-3
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ class SavedMatrixTest {
9999
assertThat(savedMatrix.billablePhysicalMinutes).isEqualTo(1)
100100
assertThat(savedMatrix.gcsPathWithoutRootBucket).isEqualTo(mockFileName)
101101
assertThat(savedMatrix.gcsRootBucket).isEqualTo(mockBucket)
102-
assertThat(savedMatrix.outcomeDetails).isNotEmpty()
102+
assertThat(savedMatrix.testAxises.first().details).isNotEmpty()
103103
}
104104

105105
@Test
@@ -133,7 +133,7 @@ class SavedMatrixTest {
133133
assertThat(savedMatrix.billablePhysicalMinutes).isEqualTo(1)
134134
assertThat(savedMatrix.gcsPathWithoutRootBucket).isEqualTo(mockFileName)
135135
assertThat(savedMatrix.gcsRootBucket).isEqualTo(mockBucket)
136-
assertThat(savedMatrix.outcomeDetails).isNotEmpty()
136+
assertThat(savedMatrix.testAxises.first().details).isNotEmpty()
137137
}
138138

139139
@Test
@@ -195,7 +195,7 @@ class SavedMatrixTest {
195195
testMatrix.state = INVALID
196196
savedMatrix = savedMatrix.updateWithMatrix(testMatrix)
197197
assertEquals(expectedOutcome, savedMatrix.outcome)
198-
assertEquals(expectedOutcomeDetails, savedMatrix.outcomeDetails)
198+
assertEquals(expectedOutcomeDetails, savedMatrix.testAxises.first().details)
199199
assertEquals(INVALID, savedMatrix.state)
200200
}
201201

0 commit comments

Comments
 (0)