diff --git a/.github/workflows/build-litertlm-native-windows.yml b/.github/workflows/build-litertlm-native-windows.yml index 32610b42..6e846735 100644 --- a/.github/workflows/build-litertlm-native-windows.yml +++ b/.github/workflows/build-litertlm-native-windows.yml @@ -80,7 +80,7 @@ jobs: Copy-Item C:\LiteRT-LM\bazel-bin\c\libLiteRtLm.dylib artifacts\LiteRtLm.dll Copy-Item StreamProxy.dll artifacts\ $prebuilt = 'C:\LiteRT-LM\prebuilt\windows_x86_64' - foreach ($base in @('libGemmaModelConstraintProvider','libLiteRt','libLiteRtTopKWebGpuSampler','libLiteRtWebGpuAccelerator')) { + foreach ($base in @('libGemmaModelConstraintProvider','libLiteRt','libLiteRtTopKWebGpuSampler','libLiteRtWebGpuAccelerator','libwebgpu_dawn')) { $src = "$prebuilt\$base.dll" if (Test-Path $src) { Copy-Item $src "artifacts\$base.dll" diff --git a/.github/workflows/build-litertlm-native.yml b/.github/workflows/build-litertlm-native.yml index b7568fe5..6c9496d1 100644 --- a/.github/workflows/build-litertlm-native.yml +++ b/.github/workflows/build-litertlm-native.yml @@ -63,6 +63,12 @@ jobs: cp /tmp/LiteRT-LM/bazel-bin/c/libLiteRtLm.dylib artifacts/libLiteRtLm.so cp /tmp/libStreamProxy.so artifacts/ cp /tmp/LiteRT-LM/prebuilt/linux_x86_64/*.so artifacts/ 2>/dev/null || true + # v0.14.0 Dawn split — libLiteRtWebGpuAccelerator.so now hard-links the + # separate libwebgpu_dawn.so. Set $ORIGIN rpath so each lib resolves its + # flat-layout siblings from its own directory at dlopen time. + sudo apt-get update -qq && sudo apt-get install -y -qq patchelf + chmod u+w artifacts/*.so # git-lfs prebuilt .so are read-only; patchelf edits in place + for f in artifacts/*.so; do patchelf --set-rpath '$ORIGIN' "$f"; done ls -lh artifacts/ - uses: actions/upload-artifact@v4 @@ -111,6 +117,12 @@ jobs: cp /tmp/LiteRT-LM/bazel-bin/c/libLiteRtLm.dylib artifacts/libLiteRtLm.so cp /tmp/libStreamProxy.so artifacts/ cp /tmp/LiteRT-LM/prebuilt/linux_arm64/*.so artifacts/ 2>/dev/null || true + # v0.14.0 Dawn split — libLiteRtWebGpuAccelerator.so now hard-links the + # separate libwebgpu_dawn.so. Set $ORIGIN rpath so each lib resolves its + # flat-layout siblings from its own directory at dlopen time. + sudo apt-get update -qq && sudo apt-get install -y -qq patchelf + chmod u+w artifacts/*.so # git-lfs prebuilt .so are read-only; patchelf edits in place + for f in artifacts/*.so; do patchelf --set-rpath '$ORIGIN' "$f"; done ls -lh artifacts/ - uses: actions/upload-artifact@v4 @@ -193,7 +205,7 @@ jobs: Copy-Item C:\LiteRT-LM\bazel-bin\c\libLiteRtLm.dylib artifacts\LiteRtLm.dll Copy-Item StreamProxy.dll artifacts\ $prebuilt = 'C:\LiteRT-LM\prebuilt\windows_x86_64' - foreach ($base in @('libGemmaModelConstraintProvider','libLiteRt','libLiteRtTopKWebGpuSampler','libLiteRtWebGpuAccelerator')) { + foreach ($base in @('libGemmaModelConstraintProvider','libLiteRt','libLiteRtTopKWebGpuSampler','libLiteRtWebGpuAccelerator','libwebgpu_dawn')) { $src = "$prebuilt\$base.dll" if (Test-Path $src) { # Keep original name for PE imports diff --git a/CLAUDE.md b/CLAUDE.md index cb10d928..256071ec 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,7 +48,7 @@ - **ModelSource**: Type-safe sealed class (`NetworkSource`, `AssetSource`, `BundledSource`, `FileSource`). See `packages/flutter_gemma/lib/core/domain/` - **Install vs Runtime separation**: Installation stores identity (modelType + fileType), runtime accepts config (maxTokens, backend, etc.) via `RuntimeConfig` - **Engine selection by file extension** (via `canHandle`): `.task`/`.bin`/`.tflite` → MediaPipe, `.litertlm` → LiteRT-LM -- **All five platforms (Android/iOS/macOS/Linux/Windows)**: Dart → `dart:ffi` → LiteRT-LM C API (inference, in `flutter_gemma_litertlm`) + LiteRT C API (embeddings, in `flutter_gemma_embeddings`). Native prebuilts fetched at build time via each package's `hook/build.dart` (Native Assets) from GitHub release `native-v0.12.0-a`. The cycle-fix `stage()` in the hooks is **Apple-only** (Xcode `directoryTreeSignature` cycle; staging on Windows splits companion DLLs and hangs cancel/close). +- **All five platforms (Android/iOS/macOS/Linux/Windows)**: Dart → `dart:ffi` → LiteRT-LM C API (inference, in `flutter_gemma_litertlm`) + LiteRT C API (embeddings, in `flutter_gemma_embeddings`). Native prebuilts fetched at build time via each package's `hook/build.dart` (Native Assets) from GitHub release `native-v0.14.0`. The cycle-fix `stage()` in the hooks is **Apple-only** (Xcode `directoryTreeSignature` cycle; staging on Windows splits companion DLLs and hangs cancel/close). ### Supported Models @@ -136,9 +136,9 @@ Core has NO pigeon (dropped at the 1.0 cut; its value types are hand-written in - **Dart SDK**: `>=3.12.0 <4.0.0` - **iOS**: Minimum 16.0 - **MediaPipe Web**: v0.10.27, Android/iOS: v0.10.33 -- **LiteRT-LM**: native libs from `native-v0.13.1-a` GitHub Release. Android tarball bundles the Qualcomm QNN dispatch stack and Windows tarball bundles Intel NPU dispatch (`LiteRtDispatch.dll` + OpenVino runtime + TBB) for `PreferredBackend.npu` (Qualcomm Snapdragon / Intel LunarLake/PantherLake). MTP (speculative decoding) support for Gemma 4 (#318 MTP crash fixed). (native-v0.13.1-a restores the NPU dispatch libs accidentally omitted from native-v0.13.1 — #155.) +- **LiteRT-LM**: native libs from `native-v0.14.0` GitHub Release. Android tarball bundles the Qualcomm QNN dispatch stack and Windows tarball bundles Intel NPU dispatch (`LiteRtDispatch.dll` + OpenVino runtime + TBB) for `PreferredBackend.npu` (Qualcomm Snapdragon / Intel LunarLake/PantherLake). v0.14.0: native per-session sampler (opaque session-config) + #214 GPU output-garbage fix; Dawn split static→dynamic (Linux/Windows bundle `libwebgpu_dawn`). Windows discrete GPU (WebGPU/Dawn) regressed upstream — use CPU/NPU on Windows (LiteRT-LM #2957). - **large_file_handler**: `^0.5.0` (core dep; 0.5.0 declares all 6 platforms — needed for pana platform support + the dart2wasm-clean web graph) -- **Current Version**: core `flutter_gemma` `1.3.1`, `flutter_gemma_rag_sqlite` `1.1.0`, `flutter_gemma_rag_qdrant` `1.1.0`; `flutter_gemma_litertlm` `1.0.4`, `flutter_gemma_mediapipe` `1.0.4`, `flutter_gemma_embeddings` `1.0.1`; `flutter_gemma_agent` `0.1.0`, `flutter_gemma_builtin_ai` `0.1.0` (new) +- **Current Version**: core `flutter_gemma` `1.3.2`, `flutter_gemma_rag_sqlite` `1.1.0`, `flutter_gemma_rag_qdrant` `1.1.0`; `flutter_gemma_litertlm` `1.2.0`, `flutter_gemma_mediapipe` `1.0.4`, `flutter_gemma_embeddings` `1.0.3`; `flutter_gemma_agent` `0.1.0`, `flutter_gemma_builtin_ai` `0.1.0` - **0.15.2**: embedding unified on LiteRT C API via Dart FFI on all native platforms (Android + iOS + Desktop). Drops `localagents-rag` JVM dep on Android and the separate TFLite C 0.12.7 tarball on Desktop; `TensorFlowLiteC` pod no longer needed on iOS. Single source of truth for `TaskType.prefix` in Dart, fixes cross-platform embedding drift (#264). ## Platform-Specific Setup @@ -175,7 +175,7 @@ window.LlmInference = LlmInference; ### Desktop (macOS/Windows/Linux) - Architecture: Dart → `dart:ffi` → LiteRT-LM C API (no JVM, no gRPC) -- Native libs fetched at build time by each package's `hook/build.dart` from `native-v0.12.0-a` GitHub release; SHA256-verified, bundled via Native Assets +- Native libs fetched at build time by each package's `hook/build.dart` from `native-v0.14.0` GitHub release; SHA256-verified, bundled via Native Assets - Desktop uses `.litertlm` format only (not `.task`) - Windows GPU requires `dxil.dll` + `dxcompiler.dll` (DirectXShaderCompiler runtime) — bundled in the Windows native archive - Windows NPU (`PreferredBackend.npu`) requires Intel LunarLake/PantherLake silicon — `LiteRtDispatch.dll` + OpenVino runtime + TBB bundled in the Windows native archive (0.15.1+) diff --git a/packages/flutter_gemma/CHANGELOG.md b/packages/flutter_gemma/CHANGELOG.md index 5afb42a5..47d2ca09 100644 --- a/packages/flutter_gemma/CHANGELOG.md +++ b/packages/flutter_gemma/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.3.2 +- Docs: add community models to the README model list (SmolLM3-3B, Phi-4-mini-reasoning, Qwen2-VL, SmolVLM2, LLaVA-OneVision). +- Docs: bump web `@litert-lm/core` setup snippet to 0.14.0. + ## 1.3.1 - Fix multi-GB background_downloader temp-file leak on Android (#383). - Deterministic download task IDs (sha256 of path triple) — stable across restarts and signed-URL rotation (#383). diff --git a/packages/flutter_gemma/DESKTOP_SUPPORT.md b/packages/flutter_gemma/DESKTOP_SUPPORT.md index 4fa9d8f4..53c08e19 100644 --- a/packages/flutter_gemma/DESKTOP_SUPPORT.md +++ b/packages/flutter_gemma/DESKTOP_SUPPORT.md @@ -2,7 +2,7 @@ Detailed setup and reference for running Flutter Gemma on **macOS, Windows, and Linux**. -> **0.14.0 architecture**: desktop platforms now run LiteRT-LM **directly via `dart:ffi`**. The previous Kotlin/JVM gRPC server (`litertlm-server.jar` + Azul Zulu JRE) is gone — no Java required, no separate process, no IPC overhead. Engine startup is ~2 s instead of ~10–15 s. +> **FFI architecture**: desktop platforms run LiteRT-LM **directly via `dart:ffi`**. The previous Kotlin/JVM gRPC server (`litertlm-server.jar` + Azul Zulu JRE) is gone — no Java required, no separate process, no IPC overhead. Engine startup is ~2 s instead of ~10–15 s. --- @@ -36,18 +36,19 @@ Detailed setup and reference for running Flutter Gemma on **macOS, Windows, and │ │ + libLiteRt.{dylib,dll,so} │ │ │ │ + libLiteRtMetalAccelerator.dylib (macOS) │ │ │ │ + libLiteRtWebGpuAccelerator.{dll,so} │ │ +│ │ + libwebgpu_dawn.{dll,so} (Linux/Windows GPU) │ │ │ │ + dxil.dll + dxcompiler.dll (Windows GPU) │ │ │ └──────────────────────────────────────────────┘ │ └──────────────────────────────────────────────────────┘ ``` **Native libraries** are fetched at build time by `hook/build.dart` from the -GitHub release `native-v0.13.1-a`, SHA256-verified, and bundled by Flutter +GitHub release `native-v0.14.0`, SHA256-verified, and bundled by Flutter [Native Assets](https://docs.flutter.dev/development/platform-integration/c-interop) into the application bundle. End-users only need to add a small -`post_install` snippet to their **macOS** `Podfile` so the bundled -companion `.framework`s get matching `lib*.dylib` symlinks for LiteRT-LM's -`gpu_registry` to find them by basename — see +`post_install` snippet to their **macOS** `Podfile` so the upstream companion +dylibs get wrapped into `.framework` bundles (and re-signed) inside the app's +`Contents/Frameworks/` for LiteRT-LM's `gpu_registry` to find them — see [macOS setup in the README](README.md#macos-setup) for the exact block. Linux and Windows are fully self-contained (no manual setup). @@ -69,11 +70,13 @@ loading sequence differs per platform (handled in `litert_lm_client.dart`). |----------|--------------|-------------|--------|-------|-------| | macOS | arm64 (Apple Silicon) | Metal | ✅ | ✅ | Vision verified on Gemma 4 + Gemma 3n via Metal | | macOS | x86_64 | — | — | — | Not supported (Apple Silicon only) | -| Windows | x86_64 | DirectX 12 (via Dawn/WebGPU) | ✅ | ✅ | Requires VS 2019+ runtime (`vcredist`) for DXC | +| Windows | x86_64 | DirectX 12 (via Dawn/WebGPU) | ✅ | ✅ | Requires VS 2019+ runtime (`vcredist`) for DXC. ⚠️ Discrete GPU regressed — see below | | Windows | arm64 | — | — | — | Not supported | | Linux | x86_64 | Vulkan (via Dawn/WebGPU) | ✅ | ✅ | glibc ≥ 2.34 (Ubuntu 22.04+, Debian 12+, RHEL 9+) | | Linux | arm64 | Vulkan (via Dawn/WebGPU) | ✅ | ✅ | Same glibc requirement | +> ⚠️ **Known regression (litertlm 1.2.0 / LiteRT-LM v0.14.0):** Windows **discrete GPUs** crash in the upstream WebGPU/Dawn stack ([LiteRT-LM #2957](https://github.com/google-ai-edge/LiteRT-LM/issues/2957)) — use `PreferredBackend.cpu` or `.npu` on Windows until upstream fixes it. macOS/Linux GPU and Windows CPU/NPU are unaffected. See [Known Limitations](#known-limitations). + For mobile platforms see the main [README](README.md). --- @@ -96,8 +99,8 @@ No Java/JVM/JRE required. ```yaml # pubspec.yaml dependencies: - flutter_gemma: ^1.2.3 # core - flutter_gemma_litertlm: ^1.0.4 # .litertlm engine — required on desktop + flutter_gemma: ^1.3.2 # core + flutter_gemma_litertlm: ^1.2.0 # .litertlm engine — required on desktop ``` ```dart @@ -148,10 +151,11 @@ For the high-level chat API with history + thinking + tool calling, use Native libs are fetched and bundled automatically via Native Assets. The **only manual step** is adding a `post_install` block to your app's -`macos/Podfile` so the bundled companion `.framework`s get matching -`lib*.dylib` symlinks (LiteRT-LM's `gpu_registry` calls -`dlopen("libLiteRtMetalAccelerator.dylib")` by basename and won't find a -bare framework binary on its own). See the +`macos/Podfile` so the upstream companion dylibs get wrapped into +`.framework` bundles (and re-signed) inside `Contents/Frameworks/`, and +`LiteRtLm.dylib`'s `LC_LOAD_DYLIB` reference is re-pointed at the new +framework path (LiteRT-LM's `gpu_registry` resolves the Metal accelerator +through that framework). See the [macOS setup snippet in the README](README.md#macos-setup) for the exact block. Without it `engine_create` returns null on `PreferredBackend.gpu` and the model silently falls back to CPU. @@ -181,6 +185,7 @@ includes: - `LiteRtLm.dll`, `LiteRt.dll`, `libGemmaModelConstraintProvider.dll` - `libLiteRtWebGpuAccelerator.dll`, `libLiteRtTopKWebGpuSampler.dll` +- `webgpu_dawn.dll` (Dawn WebGPU backend — split into a shared lib in LiteRT-LM v0.14.0; the accelerator DLL imports it, so GPU fails without it) - `dxil.dll` + `dxcompiler.dll` (DirectX Shader Compiler runtime — required for WebGPU/DX12 shader compilation; sourced from [microsoft/DirectXShaderCompiler v1.9.2602](https://github.com/microsoft/DirectXShaderCompiler/releases/tag/v1.9.2602)) @@ -203,6 +208,7 @@ The bundle includes: - `libLiteRtLm.so`, `libLiteRt.so`, `libGemmaModelConstraintProvider.so` - `libLiteRtWebGpuAccelerator.so`, `libLiteRtTopKWebGpuSampler.so`, `libStreamProxy.so` +- `libwebgpu_dawn.so` (Dawn WebGPU backend — split into a shared lib in LiteRT-LM v0.14.0; the accelerator loads it via `$ORIGIN` rpath, so GPU fails without it) `libStreamProxy.so` is a tiny helper that exposes `stream_proxy_load_global` (an `RTLD_GLOBAL` `dlopen`). The plugin uses it to pre-load `libLiteRt.so` @@ -263,7 +269,7 @@ engine with different settings results in conflicts in the GPU stack (notably a `wgpu::Instance already set` from the WebGpu sampler binary on Linux/Windows). -Since 0.14.0 the plugin avoids this by: +The plugin avoids this by: 1. Reusing the same `InferenceModel` whenever requested params match (built-in singleton in `FlutterGemmaDesktop.createModel`). 2. Disabling GPU sampler preload on Linux so the upstream sampler factory falls back to a CPU sampler — eliminates the `wgpu::Instance` conflict and re-enables runtime model swap on Linux GPU. @@ -282,6 +288,13 @@ Same as switching model — close, then reopen with the new `preferredBackend`. ## Known Limitations +### Windows discrete GPU crashes (litertlm 1.2.0 / LiteRT-LM v0.14.0) + +Windows **discrete GPUs** crash in the upstream WebGPU/Dawn stack on +`PreferredBackend.gpu` ([LiteRT-LM #2957](https://github.com/google-ai-edge/LiteRT-LM/issues/2957)). +Use `PreferredBackend.cpu` or `.npu` on Windows until upstream fixes it. +macOS/Linux GPU and Windows CPU/NPU are unaffected. + ### Per-token sampler runs on CPU on all desktop platforms When `preferredBackend: PreferredBackend.gpu`, the **forward pass** (prefill + @@ -308,115 +321,24 @@ Why this is the case: Once upstream lands the missing exports / a wgpu reset API, the plugin will re-enable GPU sampling on the affected platforms. -### `randomSeed` / `temperature` / `topK` / `topP` honoring on GPU - -**Status as of 0.14.0**: works on CPU and GPU on all platforms that ship -the patched `libLiteRtLm` build (macOS, iOS, Linux, Windows, Android). - -This required a **two-layer downstream patch** to upstream LiteRT-LM -5e0d86b applied at build time via `native/litert_lm/patch_c_api.sh`. The -patch is open-sourceable and we plan to send it upstream as a PR once it -has been validated in the wild. - -#### Why it didn't work on stock upstream - -Tracing the GPU sampler path in upstream commit 5e0d86b: - -1. **Executor hardcodes sampler params on the GPU path.** - `runtime/executor/llm_litert_compiled_model_executor.cc:1271-1279` builds - `proto::SamplerParameters` from constants (`type=TOP_P, k=1, p=0.0, - temperature=1.0, seed=0`) inside `InitializeSampler()` and passes that - to `CreateSampler(GPU, ...)`. `SessionConfig::GetSamplerParams()` is - never read by the executor. - -2. **`runtime/framework/resource_management/resource_manager.cc` would - thread params through but is dead code in OSS.** Its `BUILD` file ships - only license + visibility — no `cc_library` targets. The same file - forgets `set_seed()` during proto conversion (lines 536-562 set type, - k, p, temperature only). - -3. **GPU sampler dlopen fails on every platform**, so the factory falls - through to `CreateCpuSampler(sampler_params)` at `sampler_factory.cc:735` - — but `sampler_params` here is the hardcoded one from (1), not the - session config. - -4. **`TopPSampler::UpdateConfig` ignores seed**. `top_p_cpu_sampler.cc:168` - only mutates `k_/p_/temperature_/batch_size_`. The `std::default_random_engine` - is left as-is, so two consecutive `UpdateConfig(seed=42)` and - `UpdateConfig(seed=99)` produce different but seed-disconnected outputs. - -5. **`session_basic.cc:108` only feeds session params to the sampler when - `sampler_backend == Backend::CPU`**. The `GPU` and `NPU` branches - delegate sampler creation to the executor, which then hardcodes (1). - -So upstream is structurally seed-deaf on GPU end-to-end. Confirmation: -upstream's own commit `7ef9fee` ("Add SamplerConfig support to the Python -API and CLI") that closed [#1992](https://github.com/google-ai-edge/LiteRT-LM/issues/1992) -touched only `python/` files — zero changes to `runtime/executor/`, -`runtime/core/`, `runtime/components/`, or `c/`. **Upstream's own Python -SamplerConfig is also seed-deaf on GPU.** - -#### What our patch does - -`native/litert_lm/patch_c_api.sh` extends the upstream source with four -edits applied at build time, then runs `bazelisk build` to produce a -patched `libLiteRtLm.{so,dylib,dll}` (the same patch is also offered -upstream as -[google-ai-edge/LiteRT-LM#2080](https://github.com/google-ai-edge/LiteRT-LM/issues/2080) -and [PR #2081](https://github.com/google-ai-edge/LiteRT-LM/pull/2081)): - -- **Section 6** — `runtime/executor/llm_executor_base.h`: add - ```cpp - virtual absl::Status SetPendingSamplerParams( - const proto::SamplerParameters& sampler_params) { - return absl::UnimplementedError(...); - } - ``` - Defaulted body so executors that don't override (e.g. NPU) keep upstream - behavior — the new virtual is opt-in per executor. - -- **Section 7** — `runtime/executor/llm_litert_compiled_model_executor.h`: - add the override declaration plus a member field - `std::optional pending_sampler_params_;`. - -- **Section 8** — `runtime/executor/llm_litert_compiled_model_executor.cc`: - - Replace the hardcoded `SamplerParameters` block in `InitializeSampler` - with a `pending_sampler_params_.value_or(hardcoded_defaults)` read, - backfilling proto-zero fields so callers that pass an empty proto - don't get pathological values (e.g. `temperature=0`). - - Add the `SetPendingSamplerParams` definition. **Crucially, it does - `sampler_.reset()` unconditionally** instead of relying on - `Sampler::UpdateConfig`, because the upstream CPU sampler's - `UpdateConfig` ignores the seed (bug #4 above) and the GPU sampler - libs may not export `UpdateConfig` at all (bug #1990). Recreate-on-set - is the only reliable way to honor a fresh seed across sessions. - -- **Section 9** — `runtime/core/session_basic.cc`: invert the existing - GPU/NPU else-if to actively call `executor->SetPendingSamplerParams( - session_config.GetSamplerParams())` before the executor's first - `InitializeSampler`. `Unimplemented` returns from the base class are - silently ignored so unmodified executors keep working. - -The patch is a strict superset of upstream behavior — callers that don't -push session sampler params see no change. - -#### Validation matrix (0.14.0) - -| Platform | CPU honors seed | GPU honors seed (Strategy D) | -|---|---|---| -| macOS | ✅ | ✅ (verified Apr 27, regression_bugs_test.dart) | -| iOS (iPhone 16 Pro device) | ✅ | ✅ (validated after iOS rebuild) | -| Linux T4 / L4 VM | ✅ | ✅ (CPU sampler fallback path; UpdateConfig export confirmed) | -| Windows T4 VM | ✅ | ✅ (CPU sampler fallback) | -| Android Pixel 8 | ✅ | ✅ (UpdateConfig export confirmed in `libLiteRtTopKOpenClSampler.so`) | - -#### Tracking - -- [google-ai-edge/LiteRT-LM #1990](https://github.com/google-ai-edge/LiteRT-LM/issues/1990) — Metal sampler missing prebuilt -- [google-ai-edge/LiteRT-LM #2073](https://github.com/google-ai-edge/LiteRT-LM/issues/2073) — WebGpu sampler exports 3/7 functions -- [google-ai-edge/LiteRT-LM #1992](https://github.com/google-ai-edge/LiteRT-LM/issues/1992) (closed) — Python parity, fix didn't reach executor -- [google-ai-edge/LiteRT-LM #2080](https://github.com/google-ai-edge/LiteRT-LM/issues/2080) — bug report we filed (executor + `session_basic.cc:108` together drop GPU/NPU sampler params) -- [google-ai-edge/LiteRT-LM PR #2081](https://github.com/google-ai-edge/LiteRT-LM/pull/2081) — our proposed fix (Strategy D — `LlmExecutor::SetPendingSamplerParams` virtual + override on `LlmLiteRtCompiledModelExecutorBase` + `session_basic.cc` push). Reproducer: `flutter test integration_test/regression_bugs_test.dart` on any platform, `randomSeed=42` vs `randomSeed=99` at `temperature=1.0` on `PreferredBackend.gpu`. +### `randomSeed` / `temperature` / `topK` / `topP` honoring + +As of litertlm 1.2.0 (LiteRT-LM **v0.14.0**), per-session sampler params +(seed / temperature / topK / topP) are honored **natively** by the upstream +runtime through its opaque session config — no downstream patch. Each session +carries its own sampler params, so two sessions with different seeds produce +independent, seed-reproducible output. Verified on CPU and GPU across macOS, +iOS, Linux, Windows (CPU/NPU), and Android. + +> Earlier releases (≤ litertlm 1.1.0) needed a build-time downstream patch +> (`native/litert_lm/patch_c_api.sh`, offered upstream as +> [#2080](https://github.com/google-ai-edge/LiteRT-LM/issues/2080) / +> [PR #2081](https://github.com/google-ai-edge/LiteRT-LM/pull/2081)) because the +> stock executor hardcoded sampler params on the GPU/NPU path. v0.14.0 lands the +> native session-config sampler, so that patch is no longer applied. + +Reproducer: `flutter test integration_test/regression_bugs_test.dart` on any +platform — `randomSeed=42` vs `randomSeed=99` at `temperature=1.0`. ### Audio modality requires LiteRT-LM models @@ -445,7 +367,7 @@ your distribution's log facility. ### `glibc 2.38 not found` on Linux -The 0.14.0 bundle is built against glibc 2.34 (Ubuntu 22.04 toolchain). If +The current bundle is built against glibc 2.34 (Ubuntu 22.04 toolchain). If you see this error on a stock Ubuntu 22.04 system you're hitting a stale local binary in `native/litert_lm/prebuilt/linux_x86_64/`. Clear it: diff --git a/packages/flutter_gemma/MIGRATION.md b/packages/flutter_gemma/MIGRATION.md index a3b67a40..6db2af33 100644 --- a/packages/flutter_gemma/MIGRATION.md +++ b/packages/flutter_gemma/MIGRATION.md @@ -24,12 +24,12 @@ dependencies: **After (1.0):** ```yaml dependencies: - flutter_gemma: ^1.0.0 # core — always required - flutter_gemma_litertlm: ^1.0.0 # add if you run .litertlm models - flutter_gemma_mediapipe: ^1.0.0 # add if you run .task / .bin models - flutter_gemma_embeddings: ^1.0.0 # add if you compute embeddings - flutter_gemma_rag_qdrant: ^1.0.0 # add for native on-device RAG - flutter_gemma_rag_sqlite: ^1.0.0 # add for web (or native sqlite) RAG + flutter_gemma: ^1.3.2 # core — always required + flutter_gemma_litertlm: ^1.2.0 # add if you run .litertlm models + flutter_gemma_mediapipe: ^1.0.4 # add if you run .task / .bin models + flutter_gemma_embeddings: ^1.0.3 # add if you compute embeddings + flutter_gemma_rag_qdrant: ^1.1.0 # add for native on-device RAG + flutter_gemma_rag_sqlite: ^1.1.0 # add for web (or native sqlite) RAG ``` Pick by what you actually used in 0.16.x: diff --git a/packages/flutter_gemma/README.md b/packages/flutter_gemma/README.md index a82ff9ec..fd226c57 100644 --- a/packages/flutter_gemma/README.md +++ b/packages/flutter_gemma/README.md @@ -7,9 +7,9 @@ [![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/flutter_gemma) -**The plugin supports not only Gemma, but also other models. Here's the full list of supported models:** [Gemma 4 E2B/E4B](https://huggingface.co/litert-community/gemma-4-E2B-it-litert-lm), [Gemma3n E2B/E4B](https://huggingface.co/google/gemma-3n-E2B-it-litert-preview), [FastVLM 0.5B](https://huggingface.co/litert-community/FastVLM-0.5B), [Gemma-3 1B](https://huggingface.co/litert-community/Gemma3-1B-IT), [Gemma 3 270M](https://huggingface.co/litert-community/gemma-3-270m-it), [FunctionGemma 270M](https://huggingface.co/sasha-denisov/function-gemma-270M-it), [Qwen3 0.6B](https://huggingface.co/litert-community/Qwen3-0.6B), [Qwen 2.5](https://huggingface.co/litert-community/Qwen2.5-1.5B-Instruct), [Phi-4 Mini](https://huggingface.co/litert-community/Phi-4-mini-instruct), [DeepSeek R1](https://huggingface.co/litert-community/DeepSeek-R1-Distill-Qwen-1.5B), [SmolLM 135M](https://huggingface.co/litert-community/SmolLM-135M-Instruct). +**The plugin supports not only Gemma, but also other models. Here's the full list of supported models:** [Gemma 4 E2B/E4B](https://huggingface.co/litert-community/gemma-4-E2B-it-litert-lm), [Gemma3n E2B/E4B](https://huggingface.co/google/gemma-3n-E2B-it-litert-preview), [FastVLM 0.5B](https://huggingface.co/litert-community/FastVLM-0.5B), [Gemma-3 1B](https://huggingface.co/litert-community/Gemma3-1B-IT), [Gemma 3 270M](https://huggingface.co/litert-community/gemma-3-270m-it), [FunctionGemma 270M](https://huggingface.co/sasha-denisov/function-gemma-270M-it), [Qwen3 0.6B](https://huggingface.co/litert-community/Qwen3-0.6B), [Qwen 2.5](https://huggingface.co/litert-community/Qwen2.5-1.5B-Instruct), [Phi-4 Mini](https://huggingface.co/litert-community/Phi-4-mini-instruct), [DeepSeek R1](https://huggingface.co/litert-community/DeepSeek-R1-Distill-Qwen-1.5B), [SmolLM 135M](https://huggingface.co/litert-community/SmolLM-135M-Instruct), [SmolLM3 3B](https://huggingface.co/litert-community/SmolLM3-3B), [Phi-4 Mini Reasoning](https://huggingface.co/litert-community/Phi-4-mini-reasoning), [Qwen2-VL 2B](https://huggingface.co/litert-community/Qwen2-VL-2B), [SmolVLM2 500M](https://huggingface.co/litert-community/SmolVLM2-500M), [LLaVA-OneVision 0.5B](https://huggingface.co/litert-community/LLaVA-OneVision-0.5B). -*Note: The flutter_gemma plugin supports Gemma 4 and Gemma3n (with **multimodal vision and audio support**), FastVLM (vision), Gemma-3, FunctionGemma, Qwen3, Qwen 2.5, Phi-4, DeepSeek R1 and SmolLM. Desktop platforms (macOS, Windows, Linux) require `.litertlm` model format. +*Note: The flutter_gemma plugin supports Gemma 4 and Gemma3n (with **multimodal vision and audio support**), FastVLM, Qwen2-VL, SmolVLM2 and LLaVA-OneVision (vision), Gemma-3, FunctionGemma, Qwen3, Qwen 2.5, Phi-4 (incl. Phi-4 Mini Reasoning), DeepSeek R1, SmolLM and SmolLM3. Desktop platforms (macOS, Windows, Linux) require `.litertlm` model format. [Gemma](https://ai.google.dev/gemma) is a family of lightweight, state-of-the art open models built from the same research and technology used to create the Gemini models @@ -31,11 +31,11 @@ There is an example of using: - **Platform Support:** Compatible with iOS, Android, Web, macOS, Windows, and Linux platforms. - **🧩 Modular Packages:** A small `flutter_gemma` core plus opt-in packages — add only the engine (`.litertlm` / `.task`), embeddings, RAG, or agent code your app ships. Register them via one `FlutterGemma.initialize(...)` call. See [MIGRATION.md](MIGRATION.md). - **🖥️ Desktop Support:** Native desktop apps (macOS, Windows, Linux) with GPU acceleration via LiteRT-LM, called directly from Dart through `dart:ffi` — no JVM/JRE bundling. See [DESKTOP_SUPPORT.md](DESKTOP_SUPPORT.md) for details. -- **🖼️ Multimodal Support:** Text + Image input with Gemma 4, Gemma3n, and FastVLM vision models (all platforms incl. Web) +- **🖼️ Multimodal Support:** Text + Image input with Gemma 4, Gemma3n, FastVLM, Qwen2-VL, SmolVLM2, and LLaVA-OneVision vision models (Gemma 4 / Gemma3n on all platforms incl. Web; Qwen2-VL / SmolVLM2 / LLaVA-OneVision on Android, iOS, and Desktop; FastVLM on Desktop) - **🎙️ Audio Input:** Record and send audio messages with Gemma 4 and Gemma3n E2B/E4B models (Android, iOS device, macOS/Windows/Linux via LiteRT-LM — not on Web) - **🛠️ Function Calling:** Enable your models to call external functions and integrate with other services (supported by select models) - **🤖 On-device Agent Skills:** Opt-in [`flutter_gemma_agent`](https://pub.dev/packages/flutter_gemma_agent) — give the model `SKILL.md` skills (text / JavaScript / native-intent / MCP) it invokes through the function-calling loop, fully offline. Gallery-compatible. Android, iOS, macOS, Windows (Web not supported yet). -- **🧠 Thinking Mode:** View the reasoning process of Gemma 4, DeepSeek R1, and Qwen3 models with thinking blocks +- **🧠 Thinking Mode:** View the reasoning process of Gemma 4, DeepSeek R1, Qwen3, SmolLM3, and Phi-4 Mini Reasoning models with thinking blocks - **🛑 Stop Generation:** Cancel text generation mid-process on Android, iOS, Web, and Desktop - **⚡ Backend Switching:** Choose between CPU, GPU, and NPU backends per model — CPU/GPU on Android/iOS/Desktop, GPU on Web - **⚙️ NPU Acceleration:** Hardware NPU inference for `.litertlm` models on Qualcomm Snapdragon (Android) and Intel LunarLake/PantherLake (Windows) @@ -55,6 +55,8 @@ There is an example of using: ## What's new in 1.3 +- ⚙️ **LiteRT-LM v0.14.0 runtime** ([`flutter_gemma_litertlm`](https://pub.dev/packages/flutter_gemma_litertlm) 1.2.0) — native per-session sampler params (seed / temperature / topK / topP now honored per session) and a fix for GPU garbage output on some Android GPUs (#214). ⚠️ Known upstream regression: Windows **discrete GPUs** crash in the WebGPU/Dawn stack — use `PreferredBackend.cpu` or `.npu` on Windows (LiteRT-LM #2957). +- 🧪 **5 new community models** — SmolLM3 3B, Phi-4 Mini Reasoning, Qwen2-VL 2B, SmolVLM2 500M, LLaVA-OneVision 0.5B. - 🧠 **System OS models** — new opt-in [`flutter_gemma_builtin_ai`](https://pub.dev/packages/flutter_gemma_builtin_ai): run **Gemini Nano** (Android / AICore) and **Apple Foundation Models** (iOS 26+/macOS) with no model file to bundle or download — register `BuiltInAiEngine()` and the OS owns the weights. ## What's new in 1.2 @@ -123,7 +125,11 @@ The example app offers a curated list of models, each suited for different tasks | **Gemma 4 E4B** | Next-gen multimodal chat — text, image, audio | ✅ | ✅ | ✅ | Multilingual | 4.3GB | | **Gemma3n** | On-device multimodal chat and image analysis | ✅ | ❌ | ✅ | Multilingual | 3-6GB | | **FastVLM 0.5B** | Fast vision-language inference | ❌ | ❌ | ✅ | Multilingual | 0.5GB | +| **Qwen2-VL 2B** | Vision-language chat (image + text) | ❌ | ❌ | ✅ | Multilingual | 1.8GB | +| **SmolVLM2 500M** | Compact vision-language model | ❌ | ❌ | ✅ | Multilingual | 0.36GB | +| **LLaVA-OneVision 0.5B** | Compact vision-language model | ❌ | ❌ | ✅ | Multilingual | 0.83GB | | **Phi-4 Mini** | Advanced reasoning and instruction following | ✅ | ❌ | ❌ | Multilingual | 3.9GB | +| **Phi-4 Mini Reasoning** | Step-by-step reasoning | ❌ | ✅ | ❌ | Multilingual | 2.8GB | | **DeepSeek R1** | High-performance reasoning and code generation | ✅ | ✅ | ❌ | Multilingual | 1.7GB | | **Qwen3 0.6B** | Compact multilingual chat with function calling | ✅ | ✅ | ❌ | Multilingual | 586MB | | **Qwen 2.5** | Strong multilingual chat and instruction following | ✅ | ❌ | ❌ | Multilingual | 0.5-1.6GB | @@ -131,6 +137,7 @@ The example app offers a curated list of models, each suited for different tasks | **Gemma 3 270M** | Ideal for fine-tuning (LoRA) for specific tasks | ❌ | ❌ | ❌ | Multilingual | 0.3GB | | **FunctionGemma 270M** | Specialized for function calling on-device | ✅ | ❌ | ❌ | Multilingual | 284MB | | **SmolLM 135M** | Ultra-compact, resource-constrained devices | ❌ | ❌ | ❌ | English | 135MB | +| **SmolLM3 3B** | Multilingual small LLM with reasoning mode | ❌ | ✅ | ❌ | Multilingual | 2.0GB | | **TranslateGemma 4B** † | Single-shot 55-language translation | ❌ | ❌ | ❌ | 55 languages | 2-4GB | † **TranslateGemma is CPU-only for now.** Google hasn't released a mobile/desktop `.litertlm` bundle (HF discussion [#5](https://huggingface.co/google/translategemma-4b-it/discussions/5) — "no concrete plans"). The example app uses the community-converted bundle from [`barakplasma/translategemma-4b-it-android-task-quantized`](https://huggingface.co/barakplasma/translategemma-4b-it-android-task-quantized), which keeps `EMBEDDING_LOOKUP` weights in float32 for MediaPipe `.task` compatibility. That layout crashes the LiteRT GPU partitioner on Metal/WebGPU across all platforms — tracked upstream at [LiteRT-LM#1748](https://github.com/google-ai-edge/LiteRT-LM/issues/1748). Until Google ships the `litert-lm` quantization CLI, translation runs on CPU only (≈90 s prefill on a 4 B int4 bundle on M-series Macs). @@ -148,7 +155,7 @@ When installing models, you need to specify the correct `ModelType`. Use this ta | **Qwen 3** | `ModelType.qwen3` | Qwen3 0.6B | | **FunctionGemma** | `ModelType.functionGemma` | FunctionGemma 270M IT | | **Phi** | `ModelType.phi` | Phi-4 Mini | -| **General** | `ModelType.general` | FastVLM 0.5B, SmolLM 135M | +| **General** | `ModelType.general` | FastVLM 0.5B, SmolLM 135M, SmolLM3 3B, Phi-4 Mini Reasoning, Qwen2-VL 2B, SmolVLM2 500M, LLaVA-OneVision 0.5B | > **Note**: Gemma 4 uses `ModelType.gemma4` so its native `<\|tool_call>...` tokens are routed through the LiteRT-LM SDK's chat-template path. For Gemma 3 and earlier, keep `ModelType.gemmaIt`. @@ -367,7 +374,7 @@ script(s) for the **engine package(s) you use** to your `web/index.html`. ```html