diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d23c1d4215..16de4233da 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -155,6 +155,10 @@ jobs: path: | sdk/runanywhere-commons/dist/RACommons-android-${{ matrix.abi }}-v${{ needs.validate.outputs.version }}.zip sdk/runanywhere-commons/dist/RACommons-android-${{ matrix.abi }}-v${{ needs.validate.outputs.version }}.zip.sha256 + sdk/runanywhere-commons/dist/RABackendLLAMACPP-android-${{ matrix.abi }}-v${{ needs.validate.outputs.version }}.zip + sdk/runanywhere-commons/dist/RABackendLLAMACPP-android-${{ matrix.abi }}-v${{ needs.validate.outputs.version }}.zip.sha256 + sdk/runanywhere-commons/dist/RABackendONNX-android-${{ matrix.abi }}-v${{ needs.validate.outputs.version }}.zip + sdk/runanywhere-commons/dist/RABackendONNX-android-${{ matrix.abi }}-v${{ needs.validate.outputs.version }}.zip.sha256 retention-days: 7 if-no-files-found: error @@ -1426,6 +1430,12 @@ jobs: assert_pair "Android arm64-v8a" "RACommons-android-arm64-v8a-v${VERSION}.zip" assert_pair "Android armeabi-v7a" "RACommons-android-armeabi-v7a-v${VERSION}.zip" assert_pair "Android x86_64" "RACommons-android-x86_64-v${VERSION}.zip" + assert_pair "Android LlamaCPP arm64-v8a" "RABackendLLAMACPP-android-arm64-v8a-v${VERSION}.zip" + assert_pair "Android LlamaCPP armeabi-v7a" "RABackendLLAMACPP-android-armeabi-v7a-v${VERSION}.zip" + assert_pair "Android LlamaCPP x86_64" "RABackendLLAMACPP-android-x86_64-v${VERSION}.zip" + assert_pair "Android ONNX arm64-v8a" "RABackendONNX-android-arm64-v8a-v${VERSION}.zip" + assert_pair "Android ONNX armeabi-v7a" "RABackendONNX-android-armeabi-v7a-v${VERSION}.zip" + assert_pair "Android ONNX x86_64" "RABackendONNX-android-x86_64-v${VERSION}.zip" assert_pair "Linux x86_64" "RACommons-linux-x86_64-v${VERSION}.tar.gz" assert_pair "Windows x64" "RACommons-windows-x64-v${VERSION}.zip" assert_pair "Web WASM" "RACommons-web-v${VERSION}.tar.gz" diff --git a/sdk/runanywhere-commons/scripts/build-android.sh b/sdk/runanywhere-commons/scripts/build-android.sh index 5faa4f0938..9b7fb0db61 100755 --- a/sdk/runanywhere-commons/scripts/build-android.sh +++ b/sdk/runanywhere-commons/scripts/build-android.sh @@ -188,6 +188,35 @@ write_checksum() { fi } +create_component_archive() { + local source_dir="$1" + local component="$2" + local archive="$3" + shift 3 + + local package_parent package_root + package_parent="$(mktemp -d "${COMMONS_ROOT}/dist/android-${component}.XXXXXX")" + package_root="${package_parent}/${component}" + mkdir -p "$package_root" + stage_component "$source_dir" "$package_root" "$@" + validate_component_source "$component" "$package_root" "$@" + + mkdir -p "$(dirname "$archive")" + archive="$(cd "$(dirname "$archive")" && pwd)/$(basename "$archive")" + find "$package_root" -type d -exec chmod 0755 {} + + find "$package_root" -type f -exec chmod 0755 {} + + find "$package_root" -exec touch -h -t 198001010000 {} + + rm -f "$archive" + ( + cd "$package_parent" + LC_ALL=C find "$component" -print \ + | LC_ALL=C sort \ + | zip -qX "$archive" -@ + ) + rm -rf "$package_parent" + write_checksum "$archive" +} + main() { if [ "$#" -ne 1 ]; then echo "usage: build-android.sh " >&2 @@ -240,6 +269,12 @@ main() { create_deterministic_archive "$staging_parent" "$abi" "$archive" write_checksum "$archive" + local llamacpp_archive="${dist}/RABackendLLAMACPP-android-${abi}-v${version}.zip" + local onnx_archive="${dist}/RABackendONNX-android-${abi}-v${version}.zip" + rm -f "$llamacpp_archive" "$llamacpp_archive.sha256" "$onnx_archive" "$onnx_archive.sha256" + create_component_archive "$staging/llamacpp" "llamacpp" "$llamacpp_archive" "${LLAMACPP_LIBS[@]}" + create_component_archive "$staging/onnx" "onnx" "$onnx_archive" "${ONNX_LIBS[@]}" + echo "✓ build-android.sh complete; staged ABI '${abi}' → ${archive}" } diff --git a/sdk/runanywhere-kotlin/build.gradle.kts b/sdk/runanywhere-kotlin/build.gradle.kts index 11b670b45c..7ce41eeb6f 100644 --- a/sdk/runanywhere-kotlin/build.gradle.kts +++ b/sdk/runanywhere-kotlin/build.gradle.kts @@ -1,3 +1,4 @@ +import com.runanywhere.gradle.NativeLibraryDownload import org.gradle.api.artifacts.dsl.LockMode import org.jetbrains.kotlin.gradle.dsl.JvmTarget @@ -403,9 +404,7 @@ tasks.register("downloadJniLibs") { val tempZip = file("$tempDir/$packageName") try { - ant.withGroovyBuilder { - "get"("src" to zipUrl, "dest" to tempZip, "verbose" to false) - } + NativeLibraryDownload.downloadZipWithChecksum(zipUrl, tempZip) val extractDir = file("$tempDir/extracted-${packageName.replace(".zip", "")}") extractDir.mkdirs() @@ -424,7 +423,7 @@ tasks.register("downloadJniLibs") { tempZip.delete() } catch (e: Exception) { - logger.warn("Failed to download $packageName: ${e.message}") + throw GradleException("Failed to download and verify $packageName: ${e.message}", e) } } diff --git a/sdk/runanywhere-kotlin/buildSrc/build.gradle.kts b/sdk/runanywhere-kotlin/buildSrc/build.gradle.kts new file mode 100644 index 0000000000..e4a377cb50 --- /dev/null +++ b/sdk/runanywhere-kotlin/buildSrc/build.gradle.kts @@ -0,0 +1,8 @@ +plugins { + `kotlin-dsl` +} + +repositories { + gradlePluginPortal() + mavenCentral() +} diff --git a/sdk/runanywhere-kotlin/buildSrc/src/main/kotlin/com/runanywhere/gradle/NativeLibraryDownload.kt b/sdk/runanywhere-kotlin/buildSrc/src/main/kotlin/com/runanywhere/gradle/NativeLibraryDownload.kt new file mode 100644 index 0000000000..c1550dedce --- /dev/null +++ b/sdk/runanywhere-kotlin/buildSrc/src/main/kotlin/com/runanywhere/gradle/NativeLibraryDownload.kt @@ -0,0 +1,95 @@ +package com.runanywhere.gradle + +import org.gradle.api.GradleException +import java.io.File +import java.net.HttpURLConnection +import java.net.URI +import java.security.MessageDigest + +object NativeLibraryDownload { + private const val CONNECT_TIMEOUT_MILLIS = 30_000 + private const val READ_TIMEOUT_MILLIS = 120_000 + + fun downloadZipWithChecksum( + zipUrl: String, + destination: File, + ) { + destination.parentFile.mkdirs() + val expectedSha256 = downloadText("$zipUrl.sha256").firstSha256Token() + + try { + downloadToFile(zipUrl, destination) + val actualSha256 = destination.sha256() + if (!actualSha256.equals(expectedSha256, ignoreCase = true)) { + throw GradleException( + "SHA256 mismatch for $zipUrl: expected $expectedSha256 but downloaded $actualSha256", + ) + } + } catch (e: Exception) { + destination.delete() + throw e + } + } + + private fun downloadToFile( + url: String, + destination: File, + ) { + val connection = openHttpConnection(url) + try { + connection.inputStream.use { input -> + destination.outputStream().use { output -> + input.copyTo(output) + } + } + } finally { + connection.disconnect() + } + } + + private fun downloadText(url: String): String { + val connection = openHttpConnection(url) + return try { + connection.inputStream.bufferedReader().use { it.readText() } + } finally { + connection.disconnect() + } + } + + private fun openHttpConnection(url: String): HttpURLConnection { + val connection = URI(url).toURL().openConnection() as HttpURLConnection + connection.connectTimeout = CONNECT_TIMEOUT_MILLIS + connection.readTimeout = READ_TIMEOUT_MILLIS + connection.instanceFollowRedirects = true + connection.requestMethod = "GET" + + val responseCode = connection.responseCode + if (responseCode !in 200..299) { + connection.disconnect() + throw GradleException("Failed to download $url: HTTP $responseCode") + } + + return connection + } + + private fun String.firstSha256Token(): String { + val token = trim().split(Regex("\\s+")).firstOrNull() + if (token == null || !token.matches(Regex("[a-fA-F0-9]{64}"))) { + throw GradleException("Invalid SHA256 checksum content") + } + return token.lowercase() + } + + private fun File.sha256(): String { + val digest = MessageDigest.getInstance("SHA-256") + inputStream().use { input -> + val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + while (true) { + val read = input.read(buffer) + if (read == -1) break + digest.update(buffer, 0, read) + } + } + return digest.digest().joinToString("") { "%02x".format(it) } + } +} diff --git a/sdk/runanywhere-kotlin/gradle/verification-metadata.xml b/sdk/runanywhere-kotlin/gradle/verification-metadata.xml index d8bac6d91e..51ae6e9301 100644 --- a/sdk/runanywhere-kotlin/gradle/verification-metadata.xml +++ b/sdk/runanywhere-kotlin/gradle/verification-metadata.xml @@ -884,6 +884,9 @@ + + + @@ -1519,6 +1522,19 @@ + + + + + + + + + + + + + @@ -2520,6 +2536,19 @@ + + + + + + + + + + + + + @@ -2573,16 +2602,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + @@ -2591,11 +2644,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + @@ -2604,6 +2681,14 @@ + + + + + + + + @@ -2620,6 +2705,14 @@ + + + + + + + + @@ -2628,6 +2721,14 @@ + + + + + + + + @@ -2644,6 +2745,14 @@ + + + + + + + + @@ -2676,6 +2785,14 @@ + + + + + + + + @@ -2697,6 +2814,14 @@ + + + + + + + + @@ -2713,6 +2838,14 @@ + + + + + + + + @@ -2745,6 +2878,14 @@ + + + + + + + + @@ -2766,6 +2907,14 @@ + + + + + + + + @@ -2774,6 +2923,14 @@ + + + + + + + + @@ -2782,6 +2939,14 @@ + + + + + + + + @@ -2795,6 +2960,14 @@ + + + + + + + + @@ -2808,6 +2981,14 @@ + + + + + + + + @@ -2829,6 +3010,14 @@ + + + + + + + + @@ -2837,11 +3026,27 @@ + + + + + + + + + + + + + + + + @@ -2850,6 +3055,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + @@ -2906,6 +3135,22 @@ + + + + + + + + + + + + + + + + @@ -2922,6 +3167,14 @@ + + + + + + + + @@ -2938,6 +3191,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -3113,6 +3398,14 @@ + + + + + + + + @@ -3129,6 +3422,14 @@ + + + + + + + + @@ -3137,6 +3438,14 @@ + + + + + + + + @@ -3150,6 +3459,14 @@ + + + + + + + + diff --git a/sdk/runanywhere-kotlin/modules/runanywhere-core-llamacpp/build.gradle.kts b/sdk/runanywhere-kotlin/modules/runanywhere-core-llamacpp/build.gradle.kts index cf72fd6f7b..1a7bd6440c 100644 --- a/sdk/runanywhere-kotlin/modules/runanywhere-core-llamacpp/build.gradle.kts +++ b/sdk/runanywhere-kotlin/modules/runanywhere-core-llamacpp/build.gradle.kts @@ -1,3 +1,4 @@ +import com.runanywhere.gradle.NativeLibraryDownload import org.jetbrains.kotlin.gradle.dsl.JvmTarget plugins { @@ -189,9 +190,7 @@ tasks.register("downloadJniLibs") { val tempZip = file("$tempDir/$packageName") try { - ant.withGroovyBuilder { - "get"("src" to zipUrl, "dest" to tempZip, "verbose" to false) - } + NativeLibraryDownload.downloadZipWithChecksum(zipUrl, tempZip) val extractDir = file("$tempDir/extracted-${packageName.replace(".zip", "")}") extractDir.mkdirs() @@ -210,7 +209,9 @@ tasks.register("downloadJniLibs") { tempZip.delete() } catch (e: Exception) { - logger.warn("Failed to download $packageName: ${e.message}") + outputDir.deleteRecursively() + tempDir.deleteRecursively() + throw GradleException("Failed to download and verify $packageName: ${e.message}", e) } } diff --git a/sdk/runanywhere-kotlin/modules/runanywhere-core-onnx/build.gradle.kts b/sdk/runanywhere-kotlin/modules/runanywhere-core-onnx/build.gradle.kts index a534901a43..da3f926045 100644 --- a/sdk/runanywhere-kotlin/modules/runanywhere-core-onnx/build.gradle.kts +++ b/sdk/runanywhere-kotlin/modules/runanywhere-core-onnx/build.gradle.kts @@ -1,3 +1,4 @@ +import com.runanywhere.gradle.NativeLibraryDownload import org.jetbrains.kotlin.gradle.dsl.JvmTarget plugins { @@ -217,9 +218,7 @@ tasks.register("downloadJniLibs") { val tempZip = file("$tempDir/$packageName") try { - ant.withGroovyBuilder { - "get"("src" to zipUrl, "dest" to tempZip, "verbose" to false) - } + NativeLibraryDownload.downloadZipWithChecksum(zipUrl, tempZip) val extractDir = file("$tempDir/extracted-${packageName.replace(".zip", "")}") extractDir.mkdirs() @@ -239,7 +238,9 @@ tasks.register("downloadJniLibs") { validateAbiInventory(abi) tempZip.delete() } catch (e: Exception) { - throw GradleException("Failed to download or validate $packageName", e) + outputDir.deleteRecursively() + tempDir.deleteRecursively() + throw GradleException("Failed to download and verify $packageName: ${e.message}", e) } }