Linux iOS sharding (Flank#313)

* Linux iOS sharding

* Lazy install binaries

* Document submodule git commands

* Try loading resources from classLoader

* Fix tests

* Use folder prefix

Author: bootstraponline

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test_runner/src/main/resources/binaries"]
+	path = test_runner/src/main/resources/binaries
+	url = https://github.com/Flank/binaries.git

README.md

@@ -5,7 +5,8 @@ Flank is a [massively parallel Android and iOS test runner](https://medium.com/w
 ### Contributing
 
 - Use [JetBrains Toolbox](https://www.jetbrains.com/toolbox/app/) to install `IntelliJ IDEA Community`
-- Clone the repo `git clone https://github.com/TestArmada/flank.git`
+- Clone the repo `git clone --recursive https://github.com/TestArmada/flank.git`
+  - `git submodule update --init --recursive` updates the submodules
 - Open `test_runner/build.gradle.kts` with `IntelliJ IDEA Community`
 
 ### iOS example

test_runner/src/main/kotlin/ftl/config/FtlConstants.kt

@@ -9,6 +9,7 @@ import com.google.api.client.json.jackson2.JacksonFactory
 
 object FtlConstants {
     var useMock = false
+    val macOS = System.getProperty("os.name") == "Mac OS X"
     const val localhost = "http://localhost:8080"
 
     const val defaultIosConfig = "./flank.ios.yml"

test_runner/src/main/kotlin/ftl/ios/Parse.kt

@@ -1,10 +1,21 @@
 package ftl.ios
 
+import ftl.config.FtlConstants.macOS
 import ftl.util.Bash
+import ftl.util.Utils.copyBinaryResource
 import java.io.File
 
 object Parse {
 
+    private val installBinaries by lazy {
+        if (!macOS) {
+            copyBinaryResource("nm")
+            copyBinaryResource("swift-demangle")
+            copyBinaryResource("libatomic.so.1") // swift-demangle dependency
+            copyBinaryResource("libatomic.so.1.2.0")
+        }
+    }
+
     private fun validateFile(path: String) {
         val file = File(path)
         if (!file.exists()) {
@@ -16,16 +27,20 @@ object Parse {
 
     private fun methodName(matcher: MatchResult): String {
         return matcher.groupValues.last()
-            .replace('.', '/')
-            .replace(' ', '/')
+                .replace('.', '/')
+                .replace(' ', '/')
     }
 
     internal fun parseObjcTests(binary: String): List<String> {
+        installBinaries
         validateFile(binary)
 
         val results = mutableListOf<String>()
         // https://github.com/linkedin/bluepill/blob/37e7efa42472222b81adaa0e88f2bd82aa289b44/Source/Shared/BPXCTestFile.m#L18
-        val output = Bash.execute("nm -U $binary")
+        var cmd = "nm -U $binary"
+        if (!macOS) cmd = "PATH=~/.flank $cmd"
+        val output = Bash.execute(cmd)
+
         output.lines().forEach { line ->
             // 000089b0 t -[EarlGreyExampleTests testLayout]
             // 00008330 t -[EarlGreyExampleTests testCustomAction]
@@ -39,15 +54,22 @@ object Parse {
     }
 
     internal fun parseSwiftTests(binary: String): List<String> {
+        installBinaries
         validateFile(binary)
-
         val results = mutableListOf<String>()
-
         // The OS limits the list of arguments to ARG_MAX. Setting the xargs limit avoids a fatal
         // 'argument too long' error. xargs will split the args and run the command for each chunk.
-        val argMax = Bash.execute("getconf ARG_MAX")
+        // getconf ARG_MAX
+        val argMax = 262_144
+
+        val cmd = if (macOS) {
+            "nm -gU $binary | xargs -s $argMax xcrun swift-demangle"
+        } else {
+            "export LD_LIBRARY_PATH=~/.flank; export PATH=~/.flank:\$PATH; nm -gU $binary | xargs -s $argMax swift-demangle"
+        }
+
         // https://github.com/linkedin/bluepill/blob/37e7efa42472222b81adaa0e88f2bd82aa289b44/Source/Shared/BPXCTestFile.m#L17-18
-        val demangledOutput = Bash.execute("nm -gU $binary | xargs -s $argMax xcrun swift-demangle")
+        val demangledOutput = Bash.execute(cmd)
         demangledOutput.lines().forEach { line ->
             // _T025EarlGreyExampleTestsSwift0abceD0C10testLayoutyyF ---> EarlGreyExampleTestsSwift.EarlGreyExampleSwiftTests.testLayout() -> ()
             // _T025EarlGreyExampleTestsSwift0abceD0C16testCustomActionyyF ---> EarlGreyExampleTestsSwift.EarlGreyExampleSwiftTests.testCustomAction() -> ()

test_runner/src/main/kotlin/ftl/util/Utils.kt

@@ -95,9 +95,29 @@ object Utils {
         return bucketName.toString()
     }
 
+    private val classLoader = Thread.currentThread().contextClassLoader
+
+    private fun getResource(name: String): InputStream {
+        return classLoader.getResourceAsStream(name)
+                ?: throw RuntimeException("Unable to find resource: $name")
+    }
+
     fun readTextResource(name: String): String {
-        val resource: InputStream = this::class.java.getResourceAsStream("/$name")
-            ?: throw RuntimeException("Unable to find resource: /$name")
-        return resource.bufferedReader().use { it.readText() }
+        return getResource(name).bufferedReader().use { it.readText() }
+    }
+
+    private val userHome = System.getProperty("user.home")
+
+    fun copyBinaryResource(name: String) {
+        val destinationPath = Paths.get(userHome, ".flank", name)
+        val destinationFile = destinationPath.toFile()
+
+        if (destinationFile.exists()) return
+        destinationPath.parent.toFile().mkdirs()
+
+        // "binaries/" folder prefix is required for Linux to find the resource.
+        val bytes = getResource("binaries/$name").use { it.readBytes() }
+        Files.write(destinationPath, bytes)
+        destinationFile.setExecutable(true)
     }
 }

test_runner/src/main/resources/binaries

@@ -82,7 +82,7 @@ class ParseTest {
     }
 
     @Test(expected = RuntimeException::class)
-    fun parseSwiftTests_validateFile() {
+    fun parseSwiftTests_tmpFolder() {
         Parse.parseSwiftTests("/tmp")
     }
 }

test_runner/src/test/kotlin/ftl/ios/ParseTest.kt

@@ -10,8 +10,6 @@ import java.nio.file.Paths
 class FlankTestRunner(klass: Class<*>) : BlockJUnit4ClassRunner(klass) {
 
     companion object {
-        val macOS = System.getProperty("os.name") == "Mac OS X"
-
         init {
             println("FlankTestRunner init\n")
             val server = MockServer.application

test_runner/src/test/kotlin/ftl/test/util/FlankTestRunner.kt

@@ -1,9 +1,12 @@
 package ftl.util
 
 import com.google.common.truth.Truth.assertThat
-import ftl.test.util.FlankTestRunner.Companion.macOS
+import ftl.config.FtlConstants.macOS
+import ftl.test.util.FlankTestRunner
 import org.junit.Test
+import org.junit.runner.RunWith
 
+@RunWith(FlankTestRunner::class)
 class BashTest {
 
     @Test